<commit_before>############################################################################
#
# Copyright (C) 2014 Ruben Pollan <meskio@sindominio.net>
#
# RCCN is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# RCCN is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
############################################################################
"""
rhizomatica roaming checker
"""

from config import *

def update_foreign_subscribers():
    sub = Subscriber()
    try:
        unregistered = sub.get_all_unregistered()
        update_list(unregistered, True)
    except SubscriberException as e:
        roaming_log.error("An error ocurred getting the list of unregistered: %s" % e)

    try:
        foreign = sub.get_all_foreign()
        update_list(foreign)
    except SubscriberException as e:
        roaming_log.error("An error ocurred getting the list of unregistered: %s" % e)

def update_list(subscribers, welcome=False):
    numbering = Numbering()
    sub = Subscriber()
    for msisdn,imsi in subscribers:
        try:
            number =  numbering.get_msisdn_from_imsi(imsi)

            # check if subscriber pg_hlr[current_bts] != rk_hlr[current_bts]
            pg_hlr_current_bts = numbering.get_current_bts(number)
            rk_hlr_current_bts = numbering.get_current_bts_distributed_hlr(str(imsi))

            if pg_hlr_current_bts != rk_hlr_current_bts:
                # update subscriber location
                roaming_log.info('Subscriber %s in roaming, update location' % number)
                sub.update(msisdn, "roaming number", number)
            else:
                if welcome:
                    roaming_log.info('Subscriber %s in roaming, update location' % number)
                    sub.update(msisdn, "roaming number", number)
                    roaming_log.info('Send roaming welcome message to %s' % number)
                    send_welcome_sms(number)
                else:
                    # update only location and not the timestamp in rk_hlr
                    self.update_location(imsi, number, False)
                    roaming_log.info('Subscriber %s is roaming' % number)

        except NumberingException as e:
            roaming_log.debug("Couldn't retrieve msisdn from the imsi: %s" % e)
        except SubscriberException as e:
            roaming_log.error("An error ocurred adding the roaming number %s: %s" % (number, e))

def send_welcome_sms(number):
    try:
        sms = SMS()
        sms.send(smsc_shortcode, number, sms_welcome_roaming)
    except SMSException as e:
        roaming_log.error("An error ocurred sending welcome sms to %s: %s" % (number, e))

def update_local_subscribers():
    sub = Subscriber()
    subscribers = sub.get_all_roaming()
    for imsi in subscribers:
        try:
            msisdn = sub.get_local_msisdn(imsi)
        except SubscriberException as e:
            roaming_log.info("Couldn't retrieve the msisdn from imsi %s: %s" % (imsi, e))
            continue
        try:
            roaming_log.info('Subscriber %s is back at home_bts, update location' % msisdn)
            sub.update_location(imsi, msisdn, True)
        except SubscriberException as e:
            roaming_log.error("An error ocurred updating the location of %s: %s" % (imsi, e))

def purge_inactive_subscribers():
    sub = Subscriber()
    try:
        inactive = sub.get_all_inactive_roaming()
    except SubscriberException as e:
        roaming_log.error("An error ocurred getting the list of inactive: %s" % e)
        return

    for msisdn in inactive:
        try:
            sub.purge(msisdn)
            roaming_log.info("Roaming Subscriber %s purged" % msisdn)
        except SubscriberException as e:
            roaming_log.error("An error ocurred on %s purge: %s" % (msisdn, e))

if __name__ == '__main__':
    update_foreign_subscribers()
    update_local_subscribers()
    purge_inactive_subscribers()
<commit_msg>what a noob, object instance not self.. i'm not in a class<commit_after>############################################################################
#
# Copyright (C) 2014 Ruben Pollan <meskio@sindominio.net>
#
# RCCN is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# RCCN is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
############################################################################
"""
rhizomatica roaming checker
"""

from config import *

def update_foreign_subscribers():
    sub = Subscriber()
    try:
        unregistered = sub.get_all_unregistered()
        update_list(unregistered, True)
    except SubscriberException as e:
        roaming_log.error("An error ocurred getting the list of unregistered: %s" % e)

    try:
        foreign = sub.get_all_foreign()
        update_list(foreign)
    except SubscriberException as e:
        roaming_log.error("An error ocurred getting the list of unregistered: %s" % e)

def update_list(subscribers, welcome=False):
    numbering = Numbering()
    sub = Subscriber()
    for msisdn,imsi in subscribers:
        try:
            number =  numbering.get_msisdn_from_imsi(imsi)

            # check if subscriber pg_hlr[current_bts] != rk_hlr[current_bts]
            pg_hlr_current_bts = numbering.get_current_bts(number)
            rk_hlr_current_bts = numbering.get_current_bts_distributed_hlr(str(imsi))

            if pg_hlr_current_bts != rk_hlr_current_bts:
                # update subscriber location
                roaming_log.info('Subscriber %s in roaming, update location' % number)
                sub.update(msisdn, "roaming number", number)
            else:
                if welcome:
                    roaming_log.info('Subscriber %s in roaming, update location' % number)
                    sub.update(msisdn, "roaming number", number)
                    roaming_log.info('Send roaming welcome message to %s' % number)
                    send_welcome_sms(number)
                else:
                    # update only location and not the timestamp in rk_hlr
                    sub.update_location(imsi, number, False)
                    roaming_log.info('Subscriber %s is roaming' % number)

        except NumberingException as e:
            roaming_log.debug("Couldn't retrieve msisdn from the imsi: %s" % e)
        except SubscriberException as e:
            roaming_log.error("An error ocurred adding the roaming number %s: %s" % (number, e))

def send_welcome_sms(number):
    try:
        sms = SMS()
        sms.send(smsc_shortcode, number, sms_welcome_roaming)
    except SMSException as e:
        roaming_log.error("An error ocurred sending welcome sms to %s: %s" % (number, e))

def update_local_subscribers():
    sub = Subscriber()
    subscribers = sub.get_all_roaming()
    for imsi in subscribers:
        try:
            msisdn = sub.get_local_msisdn(imsi)
        except SubscriberException as e:
            roaming_log.info("Couldn't retrieve the msisdn from imsi %s: %s" % (imsi, e))
            continue
        try:
            roaming_log.info('Subscriber %s is back at home_bts, update location' % msisdn)
            sub.update_location(imsi, msisdn, True)
        except SubscriberException as e:
            roaming_log.error("An error ocurred updating the location of %s: %s" % (imsi, e))

def purge_inactive_subscribers():
    sub = Subscriber()
    try:
        inactive = sub.get_all_inactive_roaming()
    except SubscriberException as e:
        roaming_log.error("An error ocurred getting the list of inactive: %s" % e)
        return

    for msisdn in inactive:
        try:
            sub.purge(msisdn)
            roaming_log.info("Roaming Subscriber %s purged" % msisdn)
        except SubscriberException as e:
            roaming_log.error("An error ocurred on %s purge: %s" % (msisdn, e))

if __name__ == '__main__':
    update_foreign_subscribers()
    update_local_subscribers()
    purge_inactive_subscribers()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2018. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from argparse import ArgumentParser
import logging

from cytomine.cytomine import Cytomine, _cytomine_parameter_name_synonyms
from cytomine.models.project import Project
from cytomine.models.software import Software, Job, JobParameter, SoftwareParameterCollection
from cytomine.models.user import User

__author__ = "Begon Jean-Michel <jm.begon@gmail.com>"
__contributors = ["Mormont Romain <romainmormont@gmail.com", "Rubens Ulysse <urubens@uliege.be>"]
__copyright__ = "Copyright 2010-2018 University of Liège, Belgium, http://www.cytomine.be/"

def _inferred_number_type(v):
    """Return the inferred type for the given string. The string must contain either an interger or a float.
    """
    try:
        return int(v)
    except ValueError:
        return float(v)

def _convert_type(_type):
    # Not trying too hard for the finding types (list considered as string and number as floats)
    # Just want a first validation (when possible) with argparse.
    # Advanced checking is advised in actual job implementation.
    return {
        "Number": _inferred_number_type,
        "String": str,
        "Boolean": bool,
        "Domain": int,
        "List": str,
        "ListDomain": str,
        "Date": str
    }[_type]

def _to_bool(v):
    """
    Convert the value to boolean. Treat the following strings
    as False: {"0", "0.0", "False", "false", "FALSE"}

    Parameters
    ----------
    v: object
        A object to convert to a boolean value.

    Returns
    -------
    b: bool
        The boolean value
    """
    if isinstance(v, str):
        return v in {"0", "0.0", "False", "false", "FALSE"}
    else:
        return bool(v)

def _software_params_to_argparse(parameters):
    """
    Converts a SoftwareParameterCollection into an ArgumentParser object.

    Parameters
    ----------
    parameters: SoftwareParameterCollection
        The software parameters
    Returns
    -------
    argparse: ArgumentParser
        An initialized argument parser
    """
    # Check software parameters
    argparse = ArgumentParser()
    boolean_defaults = {}
    for parameter in parameters:
        arg_desc = {"dest": parameter.name, "required": parameter.required, "help": ""}  # TODO add help
        if parameter.type == "Boolean":
            default = _to_bool(parameter.defaultParamValue)
            arg_desc["action"] = "store_true" if not default else "store_false"
            boolean_defaults[parameter.name] = default
        else:
            python_type = _convert_type(parameter.type)
            arg_desc["type"] = python_type
            arg_desc["default"] = None if parameter.defaultParamValue is None else python_type(parameter.defaultParamValue)
        argparse.add_argument(*_cytomine_parameter_name_synonyms(parameter.name), **arg_desc)
    argparse.set_defaults(**boolean_defaults)
    return argparse

class CytomineJob(Cytomine):
    """
    ===========
    CytomineJob
    ===========
    A :class:`CytomineJob` represents a job in the cytomine context.
    This class does nothing by itself. It is just supposed to be
    inherited from or incorporated in any class representing
    a software/job registered in the Cytomine server

    Usage
    -----
    Either use the :meth:`connect`/meth:`close`methods or use it with a
    with statement:

    with CytomineJob(...) as job:
         do_your_stuff()

    Constructor parameters
    ----------------------
    software_id : int
        The identifier of the software on the Cytomine server
    project_id : int
        The identifier of the project to process on the Cytomine server
    parameters: Namespace
        A namespace mapping job parameters with their values (must not contain any other key than
        the parameters names)
    """
    def __init__(self, host, public_key, private_key, software_id, project_id, parameters=None, **kwargs):
        super(CytomineJob, self).__init__(host, public_key, private_key, **kwargs)
        self._job = None
        self._project = Project().fetch(project_id)
        self._software = Software().fetch(software_id)
        self._job_done = False
        self._parameters = parameters

    @classmethod
    def from_cli(cls, argv, **kwargs):
        # Parse CytomineJob constructor parameters
        argparse = cls._add_cytomine_cli_args(ArgumentParser())
        argparse.add_argument(*_cytomine_parameter_name_synonyms("software_id"),
                              dest="software_id", type=int, help="The Cytomine software id.", required=True)
        argparse.add_argument(*_cytomine_parameter_name_synonyms("project_id"),
                              dest="project_id", type=int, help="The Cytomine project id.", required=True)
        base_params, _ = argparse.parse_known_args(args=argv)

        log_level = base_params.verbose
        if base_params.log_level is not None:
            log_level = logging.getLevelName(base_params.log_level)

        cytomine_job = CytomineJob(
            host=base_params.host,
            public_key=base_params.public_key,
            private_key=base_params.private_key,
            software_id=base_params.software_id,
            project_id=base_params.project_id,
            parameters=None,
            verbose=log_level,
            **kwargs
        )

        # Parse and set job parameters
        params_collection = SoftwareParameterCollection(
            filters={"software": cytomine_job.software.id},
            withSetByServer=True
        ).fetch()
        cytomine_job.parameters, _ = _software_params_to_argparse(params_collection).parse_known_args(argv)
        return cytomine_job

    @property
    def job(self):
        """Return the job model

        Return
        ------
        job: cytomine.Job
            The job model
        """
        return self._job

    @property
    def project(self):
        """
        Protected method

        Return
        ------
        project_id : int
            The id of the project
        """
        return self._project

    @property
    def software(self):
        """
        Protected method

        Return
        ------
        software_id : int
            The id of the software
        """
        return self._software

    @property
    def parameters(self):
        """
        Return
        ------
        parameters : dict
            The id of the software
        """
        return self._parameters

    @parameters.setter
    def parameters(self, parameters):
        """
        Protected method.

        Parameters
        ----------
        parameters: dict
            Dictionnary mapping parameters name with their values.
        """
        self._parameters = parameters

    def done(self, status=True):
        """
        Indicates whether the job is finished or not

        Parameters
        ----------
        status : bool
            Whether the process is finished
        """
        self._job_done = status

    def is_done(self):
        """
        Return
        ------
        job_status : bool
            Whether the process is finished
        """
        return self._job_done

    def start(self):
        """
        Connect to the Cytomine server and switch to job connection
        Incurs dataflows
        """

        if not self.current_user.algo:
            # If user connects as a human (CLI execution)
            self._job = Job(self._project.id, self._software.id).save()
            user_job = User().fetch(self._job.userJob)
            self.set_credentials(user_job.publicKey, user_job.privateKey)
        else:
            # If the user executes the job through the Cytomine interface
            self._job = Job().fetch(self.current_user.job)

        # set job state to RUNNING
        self._job.status = Job.RUNNING
        self._job.update()

        # add software parameters
        if self._parameters is not None:
            parameters = vars(self._parameters)
            for software_param in self._software.parameters:
                name = software_param["name"]
                if name in parameters:
                    value = parameters[name]
                else:
                    value = software_param["defaultParamValue"]

                JobParameter(self._job.id, software_param["id"], value).save()

    def close(self, value):
        """
        Notify the Cytomine server of the job's end
        Incurs a dataflows
        """
        if value is None:
            status = Job.TERMINATED
            status_comment = "Job successfully terminated"
        else:
            status = Job.FAILED
            status_comment = str(value)[:255]

        self._job.status = status
        self._job.statusComment = status_comment
        self._job.update()

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, type, value, traceback):
        self.close(value)
        return False
<commit_msg>Prevent the creation of job parameters when job was launched from UI<commit_after># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2018. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from argparse import ArgumentParser
import logging

from cytomine.cytomine import Cytomine, _cytomine_parameter_name_synonyms
from cytomine.models.project import Project
from cytomine.models.software import Software, Job, JobParameter, SoftwareParameterCollection
from cytomine.models.user import User

__author__ = "Begon Jean-Michel <jm.begon@gmail.com>"
__contributors = ["Mormont Romain <romainmormont@gmail.com", "Rubens Ulysse <urubens@uliege.be>"]
__copyright__ = "Copyright 2010-2018 University of Liège, Belgium, http://www.cytomine.be/"

def _inferred_number_type(v):
    """Return the inferred type for the given string. The string must contain either an interger or a float.
    """
    try:
        return int(v)
    except ValueError:
        return float(v)

def _convert_type(_type):
    # Not trying too hard for the finding types (list considered as string and number as floats)
    # Just want a first validation (when possible) with argparse.
    # Advanced checking is advised in actual job implementation.
    return {
        "Number": _inferred_number_type,
        "String": str,
        "Boolean": bool,
        "Domain": int,
        "List": str,
        "ListDomain": str,
        "Date": str
    }[_type]

def _to_bool(v):
    """
    Convert the value to boolean. Treat the following strings
    as False: {"0", "0.0", "False", "false", "FALSE"}

    Parameters
    ----------
    v: object
        A object to convert to a boolean value.

    Returns
    -------
    b: bool
        The boolean value
    """
    if isinstance(v, str):
        return v in {"0", "0.0", "False", "false", "FALSE"}
    else:
        return bool(v)

def _software_params_to_argparse(parameters):
    """
    Converts a SoftwareParameterCollection into an ArgumentParser object.

    Parameters
    ----------
    parameters: SoftwareParameterCollection
        The software parameters
    Returns
    -------
    argparse: ArgumentParser
        An initialized argument parser
    """
    # Check software parameters
    argparse = ArgumentParser()
    boolean_defaults = {}
    for parameter in parameters:
        arg_desc = {"dest": parameter.name, "required": parameter.required, "help": ""}  # TODO add help
        if parameter.type == "Boolean":
            default = _to_bool(parameter.defaultParamValue)
            arg_desc["action"] = "store_true" if not default else "store_false"
            boolean_defaults[parameter.name] = default
        else:
            python_type = _convert_type(parameter.type)
            arg_desc["type"] = python_type
            arg_desc["default"] = None if parameter.defaultParamValue is None else python_type(parameter.defaultParamValue)
        argparse.add_argument(*_cytomine_parameter_name_synonyms(parameter.name), **arg_desc)
    argparse.set_defaults(**boolean_defaults)
    return argparse

class CytomineJob(Cytomine):
    """
    ===========
    CytomineJob
    ===========
    A :class:`CytomineJob` represents a job in the cytomine context.
    This class does nothing by itself. It is just supposed to be
    inherited from or incorporated in any class representing
    a software/job registered in the Cytomine server

    Usage
    -----
    Either use the :meth:`connect`/meth:`close`methods or use it with a
    with statement:

    with CytomineJob(...) as job:
         do_your_stuff()

    Constructor parameters
    ----------------------
    software_id : int
        The identifier of the software on the Cytomine server
    project_id : int
        The identifier of the project to process on the Cytomine server
    parameters: Namespace
        A namespace mapping job parameters with their values (must not contain any other key than
        the parameters names)
    """
    def __init__(self, host, public_key, private_key, software_id, project_id, parameters=None, **kwargs):
        super(CytomineJob, self).__init__(host, public_key, private_key, **kwargs)
        self._job = None
        self._project = Project().fetch(project_id)
        self._software = Software().fetch(software_id)
        self._job_done = False
        self._parameters = parameters

    @classmethod
    def from_cli(cls, argv, **kwargs):
        # Parse CytomineJob constructor parameters
        argparse = cls._add_cytomine_cli_args(ArgumentParser())
        argparse.add_argument(*_cytomine_parameter_name_synonyms("software_id"),
                              dest="software_id", type=int, help="The Cytomine software id.", required=True)
        argparse.add_argument(*_cytomine_parameter_name_synonyms("project_id"),
                              dest="project_id", type=int, help="The Cytomine project id.", required=True)
        base_params, _ = argparse.parse_known_args(args=argv)

        log_level = base_params.verbose
        if base_params.log_level is not None:
            log_level = logging.getLevelName(base_params.log_level)

        cytomine_job = CytomineJob(
            host=base_params.host,
            public_key=base_params.public_key,
            private_key=base_params.private_key,
            software_id=base_params.software_id,
            project_id=base_params.project_id,
            parameters=None,
            verbose=log_level,
            **kwargs
        )

        # Parse and set job parameters
        params_collection = SoftwareParameterCollection(
            filters={"software": cytomine_job.software.id},
            withSetByServer=True
        ).fetch()
        cytomine_job.parameters, _ = _software_params_to_argparse(params_collection).parse_known_args(argv)
        return cytomine_job

    @property
    def job(self):
        """Return the job model

        Return
        ------
        job: cytomine.Job
            The job model
        """
        return self._job

    @property
    def project(self):
        """
        Protected method

        Return
        ------
        project_id : int
            The id of the project
        """
        return self._project

    @property
    def software(self):
        """
        Protected method

        Return
        ------
        software_id : int
            The id of the software
        """
        return self._software

    @property
    def parameters(self):
        """
        Return
        ------
        parameters : dict
            The id of the software
        """
        return self._parameters

    @parameters.setter
    def parameters(self, parameters):
        """
        Protected method.

        Parameters
        ----------
        parameters: dict
            Dictionnary mapping parameters name with their values.
        """
        self._parameters = parameters

    def done(self, status=True):
        """
        Indicates whether the job is finished or not

        Parameters
        ----------
        status : bool
            Whether the process is finished
        """
        self._job_done = status

    def is_done(self):
        """
        Return
        ------
        job_status : bool
            Whether the process is finished
        """
        return self._job_done

    def start(self):
        """
        Connect to the Cytomine server and switch to job connection
        Incurs dataflows
        """

        run_by_ui = False
        if not self.current_user.algo:
            # If user connects as a human (CLI execution)
            self._job = Job(self._project.id, self._software.id).save()
            user_job = User().fetch(self._job.userJob)
            self.set_credentials(user_job.publicKey, user_job.privateKey)
        else:
            # If the user executes the job through the Cytomine interface
            self._job = Job().fetch(self.current_user.job)
            run_by_ui = True

        # set job state to RUNNING
        self._job.status = Job.RUNNING
        self._job.update()

        # add software parameters
        if not run_by_ui and self._parameters is not None:
            parameters = vars(self._parameters)
            for software_param in self._software.parameters:
                name = software_param["name"]
                if name in parameters:
                    value = parameters[name]
                else:
                    value = software_param["defaultParamValue"]

                JobParameter(self._job.id, software_param["id"], value).save()

    def close(self, value):
        """
        Notify the Cytomine server of the job's end
        Incurs a dataflows
        """
        if value is None:
            status = Job.TERMINATED
            status_comment = "Job successfully terminated"
        else:
            status = Job.FAILED
            status_comment = str(value)[:255]

        self._job.status = status
        self._job.statusComment = status_comment
        self._job.update()

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, type, value, traceback):
        self.close(value)
        return False
<|endoftext|>


<commit_before>import os
import shutil
import logging
import time
from subprocess import check_call

from pathlib import Path

from gaip import acquisition
from eodatasets import image, serialise
from eodatasets.metadata import mdf, mtl, adsfolder, image as md_image
import eodatasets.type as ptype

GA_METADATA_FILE_NAME = 'ga-metadata.yaml'

_LOG = logging.getLogger(__name__)

def _to_old_platform_names(satellite, sensor, band_number):
    """
    Translate names to those used by the old LPGS.

    Older gaip code still uses the old names.

    :type band_number: str
    :type satellite: str
    :type sensor: str
    :return: satellite, sensor, band_name
    :rtype (str, str, str)
    """
    # The sensor map uses old lpgs satellite names. Translate them to new LPGS names for now.
    if satellite == 'LANDSAT_5':
        satellite = 'Landsat5'
    if satellite == 'LANDSAT_7':
        satellite = 'Landsat7'

        if sensor == 'ETM':
            sensor = 'ETM+'

        if str(band_number) == '6_vcid_1':
            band_number = '61'
        if str(band_number) == '6_vcid_2':
            band_number = '62'

    return satellite, sensor, band_number

def expand_band_information(satellite, sensor, band_metadata, checksum=True):
    """
    Use the gaip reference table to add per-band metadata if availabe.
    :param satellite: satellite as reported by LPGS (eg. LANDSAT_8)
    :param sensor: sensor as reported by LPGS (eg. OLI_TIRS)
    :type band_metadata: ptype.BandMetadata
    :rtype: ptype.BandMetadata
    """

    satellite, sensor, band_number = _to_old_platform_names(satellite, sensor, band_metadata.number)

    bands = acquisition.SENSORS[satellite]['sensors'][sensor]['bands']

    band = bands.get(band_number)
    if band:
        band_metadata.label = band['desc']
        band_metadata.cell_size = band['resolution']
        band_metadata.type_ = band['type_desc'].lower()

    if checksum:
        _LOG.info('Checksumming band %r', band_metadata.number)
        band_metadata.checksum_md5 = image.calculate_file_md5(band_metadata.path)

    return band_metadata

def init_local_dataset(uuid=None):
    """
    Create blank metadata for a newly created dataset on this machine.
    :param uuid: The existing dataset_id, if any.
    :rtype: ptype.DatasetMetadata
    """
    md = ptype.DatasetMetadata(
        id_=uuid,
        lineage=ptype.LineageMetadata(
            machine=ptype.MachineMetadata(),
        )
    )
    return md

def create_browse_images(
        d,
        target_directory,
        rgb_bands=('6', '5', '3'),
        browse_filename='browse'):
    """

    :type d: ptype.DatasetMetadata
    :type target_directory: Path
    :type red_band_id: str
    :type green_band_id: str
    :type blue_band_id: str
    :type browse_filename: str
    :return:
    """
    red_band_id, green_band_id, blue_band_id = rgb_bands
    if d.image and d.image.bands:
        d.browse = {
            'medium': image.create_browse(
                d.image.bands[red_band_id],
                d.image.bands[green_band_id],
                d.image.bands[blue_band_id],
                target_directory / (browse_filename + '.jpg'),
                constrain_horizontal_res=1024
            ),
            'full': image.create_browse(
                d.image.bands[red_band_id],
                d.image.bands[green_band_id],
                d.image.bands[blue_band_id],
                target_directory / (browse_filename + '.fr.jpg')
            )
        }
    return d

def _copy_file(source_path, destination_path, compress_imagery=True):
    """
    Copy a file from source to destination if needed. Maybe apply compression.

    (it's generally faster to compress during a copy operation than as a separate step)

    :type source_path: Path
    :type destination_path: Path
    :type compress_imagery: bool
    :return: Size in bytes of destination file.
    :rtype int
    """
    source_size_bytes = source_path.stat().st_size

    if destination_path.exists():
        if source_path.resolve() == destination_path.resolve():
            return source_size_bytes

        if destination_path.stat().st_size == source_size_bytes:
            return source_size_bytes

    source_file = str(source_path)
    destination_file = str(destination_path)

    # Copy to destination path.

    suffix = destination_path.suffix.lower()

    # If a tif image, losslessly compress it.
    if suffix == '.tif' and compress_imagery:
        _LOG.info('Copying compressed %r -> %r', source_file, destination_file)
        check_call(
            [
                'gdal_translate',
                '--config', 'GDAL_CACHEMAX', '512',
                '--config', 'TILED', 'YES',
                '-co', 'COMPRESS=lzw',
                source_file, destination_file
            ]
        )
    else:
        _LOG.info('Copying %r -> %r', source_file, destination_file)
        shutil.copy(source_file, destination_file)

    return destination_path.stat().st_size

def prepare_target_imagery(image_directory, package_directory, compress_imagery=True, required_prefix=None):
    """
    Copy a directory of files if not already there. Possibly compress images.

    :type image_directory: Path
    :type package_directory: Path
    :return: Total size of imagery in bytes
    :rtype int
    """
    if not package_directory.exists():
        package_directory.mkdir()

    size_bytes = 0
    for source_path in image_directory.iterdir():
        # Skip hidden files and envi headers. (envi files are converted to tif during copy)
        if source_path.name.startswith('.') or source_path.suffix == '.hdr':
            continue

        destination_path = package_directory.joinpath(source_path.name)
        if destination_path.suffix == '.bin':
            destination_path = destination_path.with_suffix('.tif')

        if required_prefix and not destination_path.name.startswith(required_prefix):
            continue

        size_bytes += _copy_file(source_path, destination_path, compress_imagery)

    return size_bytes

def find_file(path, file_pattern):
    # Crude but effective. TODO: multiple/no result handling.
    return path.glob(file_pattern).next()

def do_package(metadata_extract_fn,
               image_directory,
               target_directory,
               source_datasets=None,
               rgb_bands=('6', '5', '3'),
               required_prefix=None):
    start = time.time()

    target_path = Path(target_directory).absolute()
    image_path = Path(image_directory).absolute()

    target_metadata_path = target_path / GA_METADATA_FILE_NAME
    if target_metadata_path.exists():
        _LOG.info('Already packaged? Skipping %s', target_path)
        return

    _LOG.debug('Packaging %r -> %r', image_path, target_path)
    if image_path.resolve() != target_path.resolve():
        package_directory = target_path.joinpath('package')
    else:
        package_directory = target_path

    size_bytes = prepare_target_imagery(image_path, package_directory, required_prefix=required_prefix)

    if not target_path.exists():
        target_path.mkdir()

    d = metadata_extract_fn(init_local_dataset(), package_directory)
    d.size_bytes = size_bytes

    d.lineage.source_datasets = source_datasets

    if d.image and d.image.bands:
        for number, band_metadata in d.image.bands.items():
            expand_band_information(d.platform.code, d.instrument.name, band_metadata)

    create_browse_images(d, target_path, rgb_bands=rgb_bands)

    serialise.write_yaml_metadata(d, target_metadata_path, target_path)

    _LOG.info('Packaged in %.02f: %s', time.time() - start, target_metadata_path)

def generate_ortho_metadata(d, package_directory):
    """
    :type image_directory: str
    :type target_directory: str
    :type source_datasets: dict of (str, ptype.DatasetMetadata)
    :return:
    """

    mtl_path = find_file(package_directory, '*_MTL.txt')
    _LOG.info('Reading MTL %r', mtl_path)

    return mtl.populate_from_mtl(d, mtl_path)

def generate_raw_metadata(d, package_directory):
    d = mdf.extract_md(d, package_directory)
    d = adsfolder.extract_md(d, package_directory)

    # TODO: Antenna coords for groundstation? Heading?
    # TODO: Bands?
    return d

def _find_nbar_bands(package_directory):
    bands = {}
    for band in Path(package_directory).glob('*.tif'):
        band_number = band.stem.split('_')[-1]
        bands[band_number] = ptype.BandMetadata(path=band.absolute(), number=band_number)
    return bands

def generate_nbar_metadata(d, package_directory):
    """
    :type d: ptype.DatasetMetadata
    :param d:
    :param package_directory:
    :return:
    """
    # TODO: Detect
    d.platform = ptype.PlatformMetadata(code='LANDSAT_8')
    d.instrument = ptype.InstrumentMetadata(name='OLI_TIRS')
    # d.product_type
    if not d.image:
        d.image = ptype.ImageMetadata(bands={})

    d.image.bands.update(_find_nbar_bands(package_directory))
    md_image.populate_from_image_metadata(d)
    return d

def get_dataset(directory):
    return serialise.read_yaml_metadata(find_file(directory, GA_METADATA_FILE_NAME))

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s')
    import doctest

    doctest.testmod()
    logging.getLogger().setLevel(logging.DEBUG)

    # Package RAW
    raw_ls8_dir = os.path.expanduser('~/ops/inputs/LANDSAT-8.11308/LC81160740742015089ASA00')
    do_package(generate_raw_metadata, raw_ls8_dir, raw_ls8_dir)

    # Package ORTHO, linking to previous RAW.
    ls8_packaged_out = 'out-ls8-test'
    do_package(
        generate_ortho_metadata,
        os.path.expanduser('~/ops/inputs/LS8_PINKMATTER_OUT'),
        ls8_packaged_out,
        source_datasets={'raw': get_dataset(Path(raw_ls8_dir))}
    )

    # Package NBAR, linking to previous ORTHO
    def do_no_metadata(d, package_directory):
        print 'No NBAR metadata yet'
        print d, package_directory
        return d

    do_package(
        do_no_metadata,
        os.path.expanduser('~/ops/inputs/nbar_out'),
        'out-nbar-test',
        source_datasets={
            'ortho': get_dataset(Path(ls8_packaged_out))
        }
    )

<commit_msg>Set a default creation date based on input creation date.<commit_after>import os
import shutil
import logging
import time
from subprocess import check_call
import datetime

from pathlib import Path

from gaip import acquisition
from eodatasets import image, serialise
from eodatasets.metadata import mdf, mtl, adsfolder, image as md_image
import eodatasets.type as ptype

GA_METADATA_FILE_NAME = 'ga-metadata.yaml'

_LOG = logging.getLogger(__name__)

def _to_old_platform_names(satellite, sensor, band_number):
    """
    Translate names to those used by the old LPGS.

    Older gaip code still uses the old names.

    :type band_number: str
    :type satellite: str
    :type sensor: str
    :return: satellite, sensor, band_name
    :rtype (str, str, str)
    """
    # The sensor map uses old lpgs satellite names. Translate them to new LPGS names for now.
    if satellite == 'LANDSAT_5':
        satellite = 'Landsat5'
    if satellite == 'LANDSAT_7':
        satellite = 'Landsat7'

        if sensor == 'ETM':
            sensor = 'ETM+'

        if str(band_number) == '6_vcid_1':
            band_number = '61'
        if str(band_number) == '6_vcid_2':
            band_number = '62'

    return satellite, sensor, band_number

def expand_band_information(satellite, sensor, band_metadata, checksum=True):
    """
    Use the gaip reference table to add per-band metadata if availabe.
    :param satellite: satellite as reported by LPGS (eg. LANDSAT_8)
    :param sensor: sensor as reported by LPGS (eg. OLI_TIRS)
    :type band_metadata: ptype.BandMetadata
    :rtype: ptype.BandMetadata
    """

    satellite, sensor, band_number = _to_old_platform_names(satellite, sensor, band_metadata.number)

    bands = acquisition.SENSORS[satellite]['sensors'][sensor]['bands']

    band = bands.get(band_number)
    if band:
        band_metadata.label = band['desc']
        band_metadata.cell_size = band['resolution']
        band_metadata.type_ = band['type_desc'].lower()

    if checksum:
        _LOG.info('Checksumming band %r', band_metadata.number)
        band_metadata.checksum_md5 = image.calculate_file_md5(band_metadata.path)

    return band_metadata

def init_local_dataset(uuid=None):
    """
    Create blank metadata for a newly created dataset on this machine.
    :param uuid: The existing dataset_id, if any.
    :rtype: ptype.DatasetMetadata
    """
    md = ptype.DatasetMetadata(
        id_=uuid,
        lineage=ptype.LineageMetadata(
            machine=ptype.MachineMetadata(),
        )
    )
    return md

def create_browse_images(
        d,
        target_directory,
        rgb_bands=('6', '5', '3'),
        browse_filename='browse'):
    """

    :type d: ptype.DatasetMetadata
    :type target_directory: Path
    :type red_band_id: str
    :type green_band_id: str
    :type blue_band_id: str
    :type browse_filename: str
    :return:
    """
    red_band_id, green_band_id, blue_band_id = rgb_bands
    if d.image and d.image.bands:
        d.browse = {
            'medium': image.create_browse(
                d.image.bands[red_band_id],
                d.image.bands[green_band_id],
                d.image.bands[blue_band_id],
                target_directory / (browse_filename + '.jpg'),
                constrain_horizontal_res=1024
            ),
            'full': image.create_browse(
                d.image.bands[red_band_id],
                d.image.bands[green_band_id],
                d.image.bands[blue_band_id],
                target_directory / (browse_filename + '.fr.jpg')
            )
        }
    return d

def _copy_file(source_path, destination_path, compress_imagery=True):
    """
    Copy a file from source to destination if needed. Maybe apply compression.

    (it's generally faster to compress during a copy operation than as a separate step)

    :type source_path: Path
    :type destination_path: Path
    :type compress_imagery: bool
    :return: Size in bytes of destination file.
    :rtype int
    """
    source_size_bytes = source_path.stat().st_size

    if destination_path.exists():
        if source_path.resolve() == destination_path.resolve():
            return source_size_bytes

        if destination_path.stat().st_size == source_size_bytes:
            return source_size_bytes

    source_file = str(source_path)
    destination_file = str(destination_path)

    # Copy to destination path.

    suffix = destination_path.suffix.lower()

    # If a tif image, losslessly compress it.
    if suffix == '.tif' and compress_imagery:
        _LOG.info('Copying compressed %r -> %r', source_file, destination_file)
        check_call(
            [
                'gdal_translate',
                '--config', 'GDAL_CACHEMAX', '512',
                '--config', 'TILED', 'YES',
                '-co', 'COMPRESS=lzw',
                source_file, destination_file
            ]
        )
    else:
        _LOG.info('Copying %r -> %r', source_file, destination_file)
        shutil.copy(source_file, destination_file)

    return destination_path.stat().st_size

def prepare_target_imagery(image_directory, package_directory, compress_imagery=True, required_prefix=None):
    """
    Copy a directory of files if not already there. Possibly compress images.

    :type image_directory: Path
    :type package_directory: Path
    :return: Total size of imagery in bytes
    :rtype int
    """
    if not package_directory.exists():
        package_directory.mkdir()

    size_bytes = 0
    for source_path in image_directory.iterdir():
        # Skip hidden files and envi headers. (envi files are converted to tif during copy)
        if source_path.name.startswith('.') or source_path.suffix == '.hdr':
            continue

        destination_path = package_directory.joinpath(source_path.name)
        if destination_path.suffix == '.bin':
            destination_path = destination_path.with_suffix('.tif')

        if required_prefix and not destination_path.name.startswith(required_prefix):
            continue

        size_bytes += _copy_file(source_path, destination_path, compress_imagery)

    return size_bytes

def find_file(path, file_pattern):
    # Crude but effective. TODO: multiple/no result handling.
    return path.glob(file_pattern).next()

def do_package(metadata_extract_fn,
               image_directory,
               target_directory,
               source_datasets=None,
               rgb_bands=('6', '5', '3'),
               required_prefix=None):
    start = time.time()

    target_path = Path(target_directory).absolute()
    image_path = Path(image_directory).absolute()

    target_metadata_path = target_path / GA_METADATA_FILE_NAME
    if target_metadata_path.exists():
        _LOG.info('Already packaged? Skipping %s', target_path)
        return

    _LOG.debug('Packaging %r -> %r', image_path, target_path)
    if image_path.resolve() != target_path.resolve():
        package_directory = target_path.joinpath('package')
    else:
        package_directory = target_path

    size_bytes = prepare_target_imagery(image_path, package_directory, required_prefix=required_prefix)

    if not target_path.exists():
        target_path.mkdir()

    d = metadata_extract_fn(init_local_dataset(), package_directory)
    d.size_bytes = size_bytes
    # Default creation time is creation of the source folder.
    d.creation_dt = datetime.datetime.utcfromtimestamp(image_path.stat().st_ctime)

    d.lineage.source_datasets = source_datasets

    if d.image and d.image.bands:
        for number, band_metadata in d.image.bands.items():
            expand_band_information(d.platform.code, d.instrument.name, band_metadata)

    create_browse_images(d, target_path, rgb_bands=rgb_bands)

    serialise.write_yaml_metadata(d, target_metadata_path, target_path)

    _LOG.info('Packaged in %.02f: %s', time.time() - start, target_metadata_path)

def generate_ortho_metadata(d, package_directory):
    """
    :type image_directory: str
    :type target_directory: str
    :type source_datasets: dict of (str, ptype.DatasetMetadata)
    :return:
    """

    mtl_path = find_file(package_directory, '*_MTL.txt')
    _LOG.info('Reading MTL %r', mtl_path)

    return mtl.populate_from_mtl(d, mtl_path)

def generate_raw_metadata(d, package_directory):
    d = mdf.extract_md(d, package_directory)
    d = adsfolder.extract_md(d, package_directory)

    # TODO: Antenna coords for groundstation? Heading?
    # TODO: Bands?
    return d

def _find_nbar_bands(package_directory):
    bands = {}
    for band in Path(package_directory).glob('*.tif'):
        band_number = band.stem.split('_')[-1]
        bands[band_number] = ptype.BandMetadata(path=band.absolute(), number=band_number)
    return bands

def generate_nbar_metadata(d, package_directory):
    """
    :type d: ptype.DatasetMetadata
    :param d:
    :param package_directory:
    :return:
    """
    # TODO: Detect
    d.platform = ptype.PlatformMetadata(code='LANDSAT_8')
    d.instrument = ptype.InstrumentMetadata(name='OLI_TIRS')
    # d.product_type
    if not d.image:
        d.image = ptype.ImageMetadata(bands={})

    d.image.bands.update(_find_nbar_bands(package_directory))
    md_image.populate_from_image_metadata(d)
    return d

def get_dataset(directory):
    return serialise.read_yaml_metadata(find_file(directory, GA_METADATA_FILE_NAME))

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s')
    import doctest

    doctest.testmod()
    logging.getLogger().setLevel(logging.DEBUG)

    # Package RAW
    raw_ls8_dir = os.path.expanduser('~/ops/inputs/LANDSAT-8.11308/LC81160740742015089ASA00')
    do_package(generate_raw_metadata, raw_ls8_dir, raw_ls8_dir)

    # Package ORTHO, linking to previous RAW.
    ls8_packaged_out = 'out-ls8-test'
    do_package(
        generate_ortho_metadata,
        os.path.expanduser('~/ops/inputs/LS8_PINKMATTER_OUT'),
        ls8_packaged_out,
        source_datasets={'raw': get_dataset(Path(raw_ls8_dir))}
    )

    # Package NBAR, linking to previous ORTHO
    def do_no_metadata(d, package_directory):
        print 'No NBAR metadata yet'
        print d, package_directory
        return d

    do_package(
        do_no_metadata,
        os.path.expanduser('~/ops/inputs/nbar_out'),
        'out-nbar-test',
        source_datasets={
            'ortho': get_dataset(Path(ls8_packaged_out))
        }
    )

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (C) 2015 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
import re
import collections
import shutil
import subprocess
import tempfile
import yaml

from gi.repository import GLib, Gio, GSystem

from .utils import log, fatal, run_sync, rmrf, ensuredir

def path_with_suffix(path, suffix):
    return os.path.dirname(path) + '/' + os.path.basename(path) + suffix

def make_absolute_url(parent, relpath):
    orig_parent = parent
    orig_relpath = relpath
    if parent.endswith('/'):
        parent = parent[0:-1]
    method_index = parent.find('://')
    assert method_index != -1
    first_slash = parent.find('/', method_index+3)
    assert first_slash != -1
    parent_path = parent[first_slash:]
    while relpath.startswith('../'):
        i = parent.rfind('/')
        if i == -1:
            fatal("Relative submodule path {0} is too long for parent {1}".format(orig_relpath, orig_parent))
        relpath = relpath[3:]
        parent = parent[0:i]
    parent = parent[0:first_slash] + parent
    if relpath == '':
        return parent
    return parent + '/' + relpath

GitSubmodule = collections.namedtuple('GitSubmodule',
                                      ['checksum', 'name', 'url'])

class GitMirror(object):
    _pathname_quote_re = re.compile(r'[/\.]')

    def __init__(self, mirrordir):
        self.mirrordir = mirrordir
        self.tmpdir = mirrordir + '/_tmp'
        self.gitconfig = mirrordir + '/.gitconfig'
        ensuredir(self.tmpdir)

    def _gitenv(self):
        return {'HOME': self.mirrordir}

    def _runv(self, argv, **kwargs):
        if 'env' not in kwargs:
            kwargs['env'] = {}
        kwargs['env'].update(self._gitenv())
        run_sync(['git'] + list(argv), **kwargs)

    def _run(self, *argv, **kwargs):
        self._runv(argv, **kwargs)

    def set_config(self, config):
        new = self.gitconfig + '.tmp'
        with open(config) as f:
            ygitconfig = yaml.load(f)
        with open(new, 'w') as f:
            f.write('# Automatically generated by rpmdistro-gitoverlay from gitconfig.yml; do not edit!\n')
            aliases = ygitconfig.get('aliases', [])
            for alias in aliases:
                f.write('[url "{0}"]\n  insteadof = {1}:\n'.format(alias['url'], alias['name']))
        os.rename(new, self.gitconfig)

    def _get_mirrordir(self, uri, prefix=''):
        colon = uri.find('://')
        if colon >= 0:
            scheme = uri[0:colon]
            rest = uri[colon+3:]
        else:
            raise Exception("Invalid uri {0}".format(uri))
        if prefix:
            prefix = prefix + '/'
        return self.mirrordir + '/' + prefix + scheme + '/' + rest

    def _git_revparse(self, gitdir, branch):
        return subprocess.check_output(['git', 'rev-parse', branch], cwd=gitdir).strip()

    def _strip_file_url(self, url):
        """Remove the file:// prefix, which causes git to fall back to a
        slower fetch process.

        """
        if url.startswith('file://'):
            return url[len('file://'):]
        else:
            return url

    def _list_submodules_in(self, checkout, uri, rev='HEAD'):
        self._run('checkout', '-q', '-f', rev, cwd=checkout)
        proc = subprocess.Popen(['git', 'submodule', 'status'], cwd=checkout,
                                stdout=subprocess.PIPE, env=self._gitenv())
        submodules = []
        for line in proc.stdout:
            line = line.strip()
            if line == '':
                continue
            line = line[1:]
            parts = line.split(' ', 2)
            if len(parts) < 2:
                continue
            sub_checksum, sub_name = parts[0:2]
            sub_url = subprocess.check_output(['git', 'config', '-f', '.gitmodules',
                                               'submodule.{0}.url'.format(sub_name)],
                                              cwd=checkout).strip()
            if sub_url.startswith('../'):
                sub_url = make_absolute_url(uri, sub_url)
            submodules.append(GitSubmodule(sub_checksum, sub_name, sub_url))
        return submodules

    def _list_submodules(self, gitdir, uri, branch):
        current_rev = self._git_revparse(gitdir, branch)
        tmpdir = tempfile.mkdtemp('', 'tmp-gitmirror', self.tmpdir)
        tmp_clone =  tmpdir + '/checkout'
        try:
            self._run('clone', '-q', '--no-checkout', gitdir, tmp_clone)
            return self._list_submodules_in(tmp_clone, uri, rev=branch)
        finally:
            rmrf(tmpdir)

    def mirror(self, url, branch_or_tag,
               fetch=False, fetch_continue=False):
        mirrordir = self._get_mirrordir(url)
        tmp_mirror = os.path.dirname(mirrordir) + '/' + os.path.basename(mirrordir) + '.tmp'
        did_update = False

        rmrf(tmp_mirror)
        if not os.path.isdir(mirrordir):
            self._run('clone', '--mirror', self._strip_file_url(url), tmp_mirror)
            self._run('config', 'gc.auto', '0', cwd=tmp_mirror)
            os.rename(tmp_mirror, mirrordir)
        elif fetch:
            self._run('fetch', cwd=mirrordir)

        for module in self._list_submodules(mirrordir, url, branch_or_tag):
            log("Processing {0}".format(module))
            self.mirror(module.url, module.checksum,
                        fetch=fetch, fetch_continue=fetch_continue)
        return subprocess.check_output(['git', 'rev-parse', branch_or_tag], cwd=mirrordir).strip()

    def _process_checkout_submodules(self, checkout, url):
        for module in self._list_submodules_in(checkout, url):
            sub_mirrordir = self._get_mirrordir(module.url)
            config_key = 'submodule.{0}.url'.format(module.name)
            run_sync(['git', 'config', '-f', '.gitmodules',
                      config_key, 'file://' + sub_mirrordir],
                     cwd=checkout)
            run_sync(['git', 'submodule', 'update', '--init', module.name], cwd=checkout)
            self._process_checkout_submodules(checkout + '/' + module.name, module.url)

    def checkout(self, url, branch_or_tag, dest):
        mirrordir = self._get_mirrordir(url)
        run_sync(['git', 'clone', '-s', '--origin', 'localmirror', mirrordir, dest])
        run_sync(['git', 'checkout', '-q', branch_or_tag], cwd=dest)
        self._process_checkout_submodules(dest, url)
        return dest

    def describe(self, url, branch_or_tag):
        mirrordir = self._get_mirrordir(url)
        description = subprocess.check_output(['git', 'describe', '--long', '--abbrev=40', '--always', branch_or_tag],
                                              cwd=mirrordir).strip()
        if len(description) == 40:
            return [None, description]
        else:
            rgdash = description.rfind('-g')
            assert rgdash >= 0
            return (description[0:rgdash], description[rgdash+2:])

<commit_msg>git: Add a submodule resolution cache file<commit_after>#!/usr/bin/env python
#
# Copyright (C) 2015 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
import re
import collections
import shutil
import subprocess
import tempfile
import yaml

from gi.repository import GLib, Gio, GSystem

from .utils import log, fatal, run_sync, rmrf, ensuredir

def path_with_suffix(path, suffix):
    return os.path.dirname(path) + '/' + os.path.basename(path) + suffix

def make_absolute_url(parent, relpath):
    orig_parent = parent
    orig_relpath = relpath
    if parent.endswith('/'):
        parent = parent[0:-1]
    method_index = parent.find('://')
    assert method_index != -1
    first_slash = parent.find('/', method_index+3)
    assert first_slash != -1
    parent_path = parent[first_slash:]
    while relpath.startswith('../'):
        i = parent.rfind('/')
        if i == -1:
            fatal("Relative submodule path {0} is too long for parent {1}".format(orig_relpath, orig_parent))
        relpath = relpath[3:]
        parent = parent[0:i]
    parent = parent[0:first_slash] + parent
    if relpath == '':
        return parent
    return parent + '/' + relpath

GitSubmodule = collections.namedtuple('GitSubmodule',
                                      ['checksum', 'name', 'url'])

class GitMirror(object):
    _pathname_quote_re = re.compile(r'[/\.]')

    def __init__(self, mirrordir):
        self.mirrordir = mirrordir
        self.tmpdir = mirrordir + '/_tmp'
        self.gitconfig = mirrordir + '/.gitconfig'
        ensuredir(self.tmpdir)

    def _gitenv(self):
        return {'HOME': self.mirrordir}

    def _runv(self, argv, **kwargs):
        if 'env' not in kwargs:
            kwargs['env'] = {}
        kwargs['env'].update(self._gitenv())
        run_sync(['git'] + list(argv), **kwargs)

    def _run(self, *argv, **kwargs):
        self._runv(argv, **kwargs)

    def set_config(self, config):
        new = self.gitconfig + '.tmp'
        with open(config) as f:
            ygitconfig = yaml.load(f)
        with open(new, 'w') as f:
            f.write('# Automatically generated by rpmdistro-gitoverlay from gitconfig.yml; do not edit!\n')
            aliases = ygitconfig.get('aliases', [])
            for alias in aliases:
                f.write('[url "{0}"]\n  insteadof = {1}:\n'.format(alias['url'], alias['name']))
        os.rename(new, self.gitconfig)

    def _get_mirrordir(self, uri, prefix=''):
        colon = uri.find('://')
        if colon >= 0:
            scheme = uri[0:colon]
            rest = uri[colon+3:]
        else:
            raise Exception("Invalid uri {0}".format(uri))
        if prefix:
            prefix = prefix + '/'
        return self.mirrordir + '/' + prefix + scheme + '/' + rest

    def _git_revparse(self, gitdir, branch):
        return subprocess.check_output(['git', 'rev-parse', branch], cwd=gitdir).strip()

    def _strip_file_url(self, url):
        """Remove the file:// prefix, which causes git to fall back to a
        slower fetch process.

        """
        if url.startswith('file://'):
            return url[len('file://'):]
        else:
            return url

    def _list_submodules_in(self, checkout, uri, rev='HEAD'):
        self._run('checkout', '-q', '-f', rev, cwd=checkout)
        proc = subprocess.Popen(['git', 'submodule', 'status'], cwd=checkout,
                                stdout=subprocess.PIPE, env=self._gitenv())
        submodules = []
        for line in proc.stdout:
            line = line.strip()
            if line == '':
                continue
            line = line[1:]
            parts = line.split(' ', 2)
            if len(parts) < 2:
                continue
            sub_checksum, sub_name = parts[0:2]
            sub_url = subprocess.check_output(['git', 'config', '-f', '.gitmodules',
                                               'submodule.{0}.url'.format(sub_name)],
                                              cwd=checkout).strip()
            if sub_url.startswith('../'):
                sub_url = make_absolute_url(uri, sub_url)
            submodules.append(GitSubmodule(sub_checksum, sub_name, sub_url))
        return submodules

    def _list_submodules(self, gitdir, uri, branch):
        current_rev = self._git_revparse(gitdir, branch)
        tmpdir = tempfile.mkdtemp('', 'tmp-gitmirror', self.tmpdir)
        tmp_clone =  tmpdir + '/checkout'
        try:
            self._run('clone', '-q', '--no-checkout', gitdir, tmp_clone)
            return self._list_submodules_in(tmp_clone, uri, rev=branch)
        finally:
            rmrf(tmpdir)

    def mirror(self, url, branch_or_tag,
               fetch=False, fetch_continue=False):
        mirrordir = self._get_mirrordir(url)
        tmp_mirror = os.path.dirname(mirrordir) + '/' + os.path.basename(mirrordir) + '.tmp'
        did_update = False

        rmrf(tmp_mirror)
        if not os.path.isdir(mirrordir):
            self._run('clone', '--mirror', self._strip_file_url(url), tmp_mirror)
            self._run('config', 'gc.auto', '0', cwd=tmp_mirror)
            os.rename(tmp_mirror, mirrordir)
        elif fetch:
            self._run('fetch', cwd=mirrordir)

        rev = subprocess.check_output(['git', 'rev-parse', branch_or_tag], cwd=mirrordir).strip()

        # Cache making it more efficient to remirror the same commit
        # multiple times
        cachepath = mirrordir + '/submodules-cache-stamp'
        if os.path.exists(cachepath):
            cached_rev = open(cachepath).read().strip()
            if cached_rev == rev:
                return rev

        for module in self._list_submodules(mirrordir, url, branch_or_tag):
            log("Processing {0}".format(module))
            self.mirror(module.url, module.checksum,
                        fetch=fetch, fetch_continue=fetch_continue)
        with open(cachepath + '.tmp', 'w') as f:
            f.write(rev + '\n')
        os.rename(cachepath + '.tmp', cachepath)
        return rev

    def _process_checkout_submodules(self, checkout, url):
        for module in self._list_submodules_in(checkout, url):
            sub_mirrordir = self._get_mirrordir(module.url)
            config_key = 'submodule.{0}.url'.format(module.name)
            run_sync(['git', 'config', '-f', '.gitmodules',
                      config_key, 'file://' + sub_mirrordir],
                     cwd=checkout)
            run_sync(['git', 'submodule', 'update', '--init', module.name], cwd=checkout)
            self._process_checkout_submodules(checkout + '/' + module.name, module.url)

    def checkout(self, url, branch_or_tag, dest):
        mirrordir = self._get_mirrordir(url)
        run_sync(['git', 'clone', '-s', '--origin', 'localmirror', mirrordir, dest])
        run_sync(['git', 'checkout', '-q', branch_or_tag], cwd=dest)
        self._process_checkout_submodules(dest, url)
        return dest

    def describe(self, url, branch_or_tag):
        mirrordir = self._get_mirrordir(url)
        description = subprocess.check_output(['git', 'describe', '--long', '--abbrev=40', '--always', branch_or_tag],
                                              cwd=mirrordir).strip()
        if len(description) == 40:
            return [None, description]
        else:
            rgdash = description.rfind('-g')
            assert rgdash >= 0
            return (description[0:rgdash], description[rgdash+2:])

<|endoftext|>


<commit_before>import numpy as np
import cudatree
import time

inputs = []
best_threshold_prcts = []
best_threshold_values = []

all_classes = [2, 10, 50, 100]
all_examples = [2*10**4, 5*10**4, 25*10**4]
all_features = [8, 16, 32, 64, 512]
thresholds = [1000, 2000, 3000, 4000, 5000, 10000,15000,20000,25000,30000]
total_iters = len(all_classes) * len(all_examples) * len(all_features) * len(thresholds)
i = 1
# thresholds =  [0.001, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, .1, .2]
for n_classes in all_classes:
  print "n_classes", n_classes
  for n_examples in all_examples:
    print "n_examples", n_examples
    y = np.random.randint(low = 0, high = n_classes, size = n_examples)
    for n_features in all_features:
      print "n_features", n_features
      max_features = int(np.sqrt(n_features))
      print "sqrt(n_features) =", max_features
      if n_features * n_examples > 100 * 10**6:
        print "Skipping due excessive n_features * n_examples..."
        continue
      if n_examples * n_classes > 10 ** 8:
        print "Skipping due to excessive n_examples * n_classes"
        continue

      x = np.random.randn(n_examples, n_features)
      rf = cudatree.RandomForestClassifier(n_estimators = 3, bootstrap = False, max_features = max_features)
      # warm up
      rf.fit(x[:100],y[:100])
      best_time = np.inf
      best_threshold = None
      best_threshold_prct = None
      for bfs_threshold in thresholds:

        bfs_threshold_prct = float(bfs_threshold) / n_examples
        print "  -- (%d / %d) threshold %d (%0.2f%%)" % (i, total_iters,  bfs_threshold, bfs_threshold_prct * 100)
        i += 1
        start_t = time.time()
        rf.fit(x, y, bfs_threshold)
        t = time.time() - start_t
        print "  ---> total time", t
        if t < best_time:
          best_time = t
          best_threshold = bfs_threshold
          best_theshold_prct = bfs_threshold_prct

      inputs.append([1.0, n_classes, n_examples, max_features])
      best_threshold_values.append(best_threshold)
      best_threshold_prcts.append(best_threshold_prct)

X = np.array(inputs)
print "input shape", X.shape
# n_samples / 1000
X[:, 2] /= 1000.0

best_threshold_prcts = np.array(best_threshold_prcts)
best_threshold_values = np.array(best_threshold_values)
Y = best_threshold_values / 1000.0

lstsq_result = np.linalg.lstsq(X, Y)
print "Regression coefficients:", lstsq_result[0]
n = len(best_threshold_values)
print "Regression residual:", lstsq_result[1], "RMSE:", np.sqrt(lstsq_result[1] / n)

import socket
csv_filename = "threshold_results_" + socket.gethostname()
with open(csv_filename, 'w') as csvfile:
    for i, input_tuple in enumerate(inputs):
      csvfile.write(str(input_tuple[1:]))
      csvfile.write("," + str(best_threshold_values[i]))
      csvfile.write("," + str(best_threshold_prcts[i]))
      csvfile.write("\n")

import sklearn
import sklearn.linear_model
ridge = sklearn.linear_model.RidgeCV(alphas = [0.01, 0.1, 1, 10, 100], fit_intercept = False)
ridge.fit(X, Y)
print "Ridge regression coef", ridge.coef_
print "Ridge regression alpha", ridge.alpha_

pred = ridge.predict(X)
sse = np.sum( (pred - best_threshold_values) ** 2)
print "Ridge residual", sse
print "Ridge RMSE", np.sqrt(sse / n)
<commit_msg>estimator<commit_after>import numpy as np
import cudatree
import time

inputs = []
best_threshold_prcts = []
best_threshold_values = []

all_classes = [2, 10, 50, 100]
all_examples = [2*10**4, 5*10**4, 25*10**4]
all_features = [8, 16, 32, 64, 512]
thresholds = [1000, 2000, 3000, 4000, 5000, 10000,15000,20000,25000,30000]
total_iters = len(all_classes) * len(all_examples) * len(all_features) * len(thresholds)
i = 1
# thresholds =  [0.001, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, .1, .2]
for n_classes in all_classes:
  print "n_classes", n_classes
  for n_examples in all_examples:
    print "n_examples", n_examples
    y = np.random.randint(low = 0, high = n_classes, size = n_examples)
    for n_features in all_features:
      print "n_features", n_features
      max_features = int(np.sqrt(n_features))
      print "sqrt(n_features) =", max_features
      if n_features * n_examples > 100 * 10**6:
        print "Skipping due excessive n_features * n_examples..."
        continue
      if n_examples * n_classes > 10 ** 8:
        print "Skipping due to excessive n_examples * n_classes"
        continue

      x = np.random.randn(n_examples, n_features)
      rf = cudatree.RandomForestClassifier(n_estimators = 3, bootstrap = False, max_features = max_features)
      # warm up
      rf.fit(x[:100],y[:100])
      best_time = np.inf
      best_threshold = None
      best_threshold_prct = None
      for bfs_threshold in thresholds:

        bfs_threshold_prct = float(bfs_threshold) / n_examples
        print "  -- (%d / %d) threshold %d (%0.2f%%)" % (i, total_iters,  bfs_threshold, bfs_threshold_prct * 100)
        i += 1
        start_t = time.time()
        rf.fit(x, y, bfs_threshold)
        t = time.time() - start_t
        print "  ---> total time", t
        if t < best_time:
          best_time = t
          best_threshold = bfs_threshold
          best_theshold_prct = bfs_threshold_prct

      inputs.append([1.0, n_classes, n_examples, max_features])
      best_threshold_values.append(best_threshold)
      best_threshold_prcts.append(best_threshold_prct)

X = np.array(inputs)
print "input shape", X.shape
# n_samples / 1000
X[:, 2] /= 1000.0

best_threshold_prcts = np.array(best_threshold_prcts)
best_threshold_values = np.array(best_threshold_values)
Y = best_threshold_values / 1000.0

lstsq_result = np.linalg.lstsq(X, Y)
print "Regression coefficients:", lstsq_result[0]
n = len(best_threshold_values)
print "Regression residual:", lstsq_result[1], "RMSE:", np.sqrt(lstsq_result[1] / n)

import socket
csv_filename = "threshold_results_" + socket.gethostname()
with open(csv_filename, 'w') as csvfile:
    for i, input_tuple in enumerate(inputs):
      csvfile.write(str(input_tuple[1:]))
      csvfile.write("," + str(best_threshold_values[i]))
      csvfile.write("," + str(best_threshold_prcts[i]))
      csvfile.write("\n")

import sklearn
import sklearn.linear_model
ridge = sklearn.linear_model.RidgeCV(alphas = [0.01, 0.1, 1, 10, 100], fit_intercept = False)
ridge.fit(X, Y)
print "Ridge regression coef", ridge.coef_
print "Ridge regression alpha", ridge.alpha_

pred = ridge.predict(X)
sse = np.sum( (pred - Y) ** 2)
print "Ridge residual", sse
print "Ridge RMSE", np.sqrt(sse / n)
<|endoftext|>


<commit_before>import pygatt
import time

def tempConvert(rawObjTemp, rawAmbTemp):
    SCALE_LSB = 0.03125
    t = rawObjTemp >> 2
    obj = t * SCALE_LSB
    t = rawAmbTemp >> 2
    amb = t * SCALE_LSB
    return [obj, amb]

def humConvert(rawTemp, rawHum):
    temp = (float(rawTemp) / 65536) * 165 - 40
    hum = (float(rawHum) / 65536) * 100
    return [temp, hum]

adapter = pygatt.GATTToolBackend()

try:
    adapter.start()
    device = adapter.connect('24:71:89:BC:84:84')
    device.char_write('f000aa02-0451-4000-b000-000000000000', bytearray([0x01]))
    device.char_write('f000aa22-0451-4000-b000-000000000000', bytearray([0x01]))
    time.sleep(2)
    while True:
        valueTemp = device.char_read('f000aa01-0451-4000-b000-000000000000')
        valueHum = device.char_read('f000aa21-0451-4000-b000-000000000000')
        bytes = []
        for x in valueTemp: bytes.append("{:02x}".format(x))
        rawObjTemp = int('0x' + bytes[1] + bytes[0], 16)
        rawAmbTemp = int('0x' + bytes[3] + bytes[2], 16)
        resultsTemp = tempConvert(rawObjTemp, rawAmbTemp)
        bytes = []
        for x in valueHum: bytes.append("{:02x}".format(x))
        rawTemp = int ('0x' + bytes[1] + bytes[0], 16)
        rawHum = int ('0x' + bytes[3] + bytes[2], 16)
        resultsHum = humConvert(rawTemp, rawHum)
        print "Obj: {:.1f} Amb: {:.1f}".format(resultsTemp[0], resultsTemp[1])
        print "Tmp: {:.1f} Hum: {:.1f}".format(resultsHum[0], resultsHum[1])
        time.sleep(1)
finally:
    adapter.stop()
<commit_msg>More useful output<commit_after>import pygatt
import time

def tempConvert(rawObjTemp, rawAmbTemp):
    SCALE_LSB = 0.03125
    t = rawObjTemp >> 2
    obj = t * SCALE_LSB
    t = rawAmbTemp >> 2
    amb = t * SCALE_LSB
    return [obj, amb]

def humConvert(rawTemp, rawHum):
    temp = (float(rawTemp) / 65536) * 165 - 40
    hum = (float(rawHum) / 65536) * 100
    return [temp, hum]

MAC = '24:71:89:BC:84:84'
adapter = pygatt.GATTToolBackend()

try:
    adapter.start()
    print "Connecting to " + MAC
    device = adapter.connect(MAC)
    print "Connected. Enabling sensors"
    device.char_write('f000aa02-0451-4000-b000-000000000000', bytearray([0x01]))
    device.char_write('f000aa22-0451-4000-b000-000000000000', bytearray([0x01]))
    print "Sensors enabled. Starting measurements"
    time.sleep(2)
    print "\n"
    while True:
        valueTemp = device.char_read('f000aa01-0451-4000-b000-000000000000')
        valueHum = device.char_read('f000aa21-0451-4000-b000-000000000000')
        bytes = []
        for x in valueTemp: bytes.append("{:02x}".format(x))
        rawObjTemp = int('0x' + bytes[1] + bytes[0], 16)
        rawAmbTemp = int('0x' + bytes[3] + bytes[2], 16)
        resultsTemp = tempConvert(rawObjTemp, rawAmbTemp)
        bytes = []
        for x in valueHum: bytes.append("{:02x}".format(x))
        rawTemp = int ('0x' + bytes[1] + bytes[0], 16)
        rawHum = int ('0x' + bytes[3] + bytes[2], 16)
        resultsHum = humConvert(rawTemp, rawHum)
        print "\033[3A"
        print "Obj: {:.1f} Amb: {:.1f}".format(resultsTemp[0], resultsTemp[1])
        print "Tmp: {:.1f} Hum: {:.1f}".format(resultsHum[0], resultsHum[1])
        time.sleep(1)
finally:
    adapter.stop()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding:utf-8
#
# Copyright [2015] [Yoshihiro Tanaka]
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__Author__ =  "Yoshihiro Tanaka"
__date__   =  "2015-02-17"

import json, sys, os, time

def main():
    bef = []
    beftime = 0.0
    while True:
        filepath = '/home/compbio/.mozilla/firefox/2rx2yf69.default/sessionstore-backups/recovery.js'
        mtime = os.stat(filepath).st_mtime
        if mtime != beftime:
            f = open(filepath)
            data = json.load(f)
            readdata = data['windows'][0]['cookies'][0]['value']
            items = readdata.split('%7C')

            if bef != items:
                writeFile(items[3:5])
                writeFile(items[10:12])
                bef = items
                print "write to file."
        beftime = mtime
        time.sleep(1.0)

def writeFile(items):
    with open('girls.tsv', 'a') as fw:
        fw.write('\t'.join(items) + '\n')

if __name__=='__main__':
    os.system('touch girls.tsv')
    main()
<commit_msg>fixed filepath<commit_after>#!/usr/bin/env python
# encoding:utf-8
#
# Copyright [2015] [Yoshihiro Tanaka]
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__Author__ =  "Yoshihiro Tanaka"
__date__   =  "2015-02-17"

import json, sys, os, time

def main():
    bef = []
    beftime = 0.0
    home = os.path.expanduser("~")
    while True:
        filepath = os.path.join(home, '.mozilla/firefox/2rx2yf69.default/sessionstore-backups/recovery.js')
        if os.path.exists(filepath):
            mtime = os.stat(filepath).st_mtime
            if mtime != beftime:
                f = open(filepath)
                data = json.load(f)
                readdata = data['windows'][0]['cookies'][0]['value']
                items = readdata.split('%7C')

                if bef != items:
                    writeFile(items[3:5])
                    writeFile(items[10:12])
                    bef = items
                    print "write to file."
            beftime = mtime
        else:
            sys.stderr.write("file not found.\n")
        time.sleep(1.0)

def writeFile(items):
    with open('girls.tsv', 'a') as fw:
        fw.write('\t'.join(items) + '\n')

if __name__=='__main__':
    os.system('touch girls.tsv')
    main()
<|endoftext|>


<commit_before>import argparse
import csv
import gmail
import logging
from nltk.corpus import stopwords
import operator
import sys
from textblob import TextBlob

logging.basicConfig(stream=sys.stdout, level=logging.WARN)
logger = logging.getLogger("Gmail Reader")

try:
    from credentials import username, password
except ImportError:
    logger.error(
        """You need to create a file named 'credentials.py' in this folder with two variables: username and password.
If you use two-factor authentication with Gmail, you will need to create an application-specific password to use here
as your normal password will not work. See https://support.google.com/accounts/answer/185833?hl=en for more.""")

def get_bodies_from_label(label):
    logger.info("Opening mail in %s", label)
    all_mail = []
    mail_bodies = []
    try:
        all_mail = g.label(label).mail(prefetch=True, skip_attachments=True)
    except AttributeError:
        logger.error("'%s' does not appear to be a valid label-- please check the spelling and case", label)
    for email in all_mail:
        logger.info(u"Reading %s", email.subject)
        try:
            body = email.body.decode("utf8")
        except UnicodeDecodeError:
            logger.warn(u"Skipping %s due to Unicode issues", email.subject)
            continue
        if body:
            mail_bodies.append(body)
    return mail_bodies

def write_counts_to_csv(bodies, filename):
    all_email_bodies = TextBlob(" ".join(bodies))
    interesting = {word: count for word, count in all_email_bodies.word_counts.items() if word not in stops}
    sorted_counts = sorted(interesting.items(), key=operator.itemgetter(1), reverse=True)
    filename = "%s.csv" % filename
    with open(filename, "wb") as f:
        logger.warn("Writing to %s", filename)
        writer = csv.writer(f)
        writer.writerow(["Word", "Count"])
        for word, count in sorted_counts:
            writer.writerow([word, count])

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("labels", nargs="*", help="one or more Gmail labels separated by spaces")
    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="show all logging information")
    args = parser.parse_args()
    if args.verbose:
        logger.level = logging.DEBUG
    try:
        g = gmail.login(username, password)
    except gmail.AuthenticationError:
        logger.error("Unable to connect to the %s account. You may need to use an application-specific password.",
                     username)
        sys.exit(1)
    mail_bodies = []
    try:
        stops = set(stopwords.words("english"))
    except LookupError:
        import nltk
        nltk.download("stopwords")
        logger.warn("Need to download NLTK data for stopwords. This should only happen the first time you run me.")
        stops = set(stopwords.words("english"))

    for label in args.labels:
        mail_bodies += get_bodies_from_label(label)
    write_counts_to_csv(mail_bodies, "-".join(args.labels))
    g.logout()
<commit_msg>Skip empty/ non-existent mail<commit_after>import argparse
import csv
import gmail
import logging
from nltk.corpus import stopwords
import operator
import sys
from textblob import TextBlob

logging.basicConfig(stream=sys.stdout, level=logging.WARN)
logger = logging.getLogger("Gmail Reader")

try:
    from credentials import username, password
except ImportError:
    logger.error(
        """You need to create a file named 'credentials.py' in this folder with two variables: username and password.
If you use two-factor authentication with Gmail, you will need to create an application-specific password to use here
as your normal password will not work. See https://support.google.com/accounts/answer/185833?hl=en for more.""")

def get_bodies_from_label(label):
    logger.info("Opening mail in %s", label)
    all_mail = []
    mail_bodies = []
    try:
        all_mail = g.label(label).mail(prefetch=True, skip_attachments=True)
    except AttributeError:
        logger.error("'%s' does not appear to be a valid label-- please check the spelling and case", label)
    for email in all_mail:
        logger.info(u"Reading %s", email.subject)
        if not email.body:
            logger.warn(u"Skipping %s as it seems to be empty", email.subject)
            continue
        try:
            body = email.body.decode("utf8")
        except UnicodeDecodeError:
            logger.warn(u"Skipping %s due to Unicode issues", email.subject)
            continue
        if body:
            mail_bodies.append(body)
    return mail_bodies

def write_counts_to_csv(bodies, filename):
    all_email_bodies = TextBlob(" ".join(bodies))
    interesting = {word: count for word, count in all_email_bodies.word_counts.items() if word not in stops}
    sorted_counts = sorted(interesting.items(), key=operator.itemgetter(1), reverse=True)
    filename = "%s.csv" % filename
    with open(filename, "wb") as f:
        logger.warn("Writing to %s", filename)
        writer = csv.writer(f)
        writer.writerow(["Word", "Count"])
        for word, count in sorted_counts:
            writer.writerow([word, count])

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("labels", nargs="*", help="one or more Gmail labels separated by spaces")
    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="show all logging information")
    args = parser.parse_args()
    if args.verbose:
        logger.level = logging.DEBUG
    try:
        g = gmail.login(username, password)
    except gmail.AuthenticationError:
        logger.error("Unable to connect to the %s account. You may need to use an application-specific password.",
                     username)
        sys.exit(1)
    mail_bodies = []
    try:
        stops = set(stopwords.words("english"))
    except LookupError:
        import nltk
        nltk.download("stopwords")
        logger.warn("Need to download NLTK data for stopwords. This should only happen the first time you run me.")
        stops = set(stopwords.words("english"))

    for label in args.labels:
        mail_bodies += get_bodies_from_label(label)
    write_counts_to_csv(mail_bodies, "-".join(args.labels))
    g.logout()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#game.py
import PythonLibraries.matrix_lib as matrix_lib

class data_square(object):
	def __init__(self, tokens):
		self.data = {}
		self.total = 0
		for t in tokens:
			d = {0:0, 1:0, 2:0}
			self.data[t] = d

	def get_prob(self, token, winner):
		num = 1
		if self.data[token][winner] != 0:
			num = self.data[token][winner]
		return num / float(self.total)

	def get_total(self):
		return self.total

	def get_total_by_token(self,token):
		return sum(self.data[token].values())

	def get_total_by_winner(self, winner):
		return sum([x[winner] for x in self.data.values()])

	def add_game(self, token, winner):
		self.data[token][winner] += 1
		self.total += 1

class Recorder(object):
	def __init__(self, dataFilename, num_rows, num_cols, tokens):
		self.filename = dataFilename
		self.tokens = tokens
		self.rows = num_rows
		self.cols = num_cols
		self.matrix = []
		for y in range(self.rows):
			l = []
			for x in range(self.cols):
				l.append(data_square(self.tokens))
			self.matrix.append(l)
		f = open(self.filename, 'r')
		for line in f:
			self.parse_game(line)
		f.close()

	def parse_game(self, game_state):
		grid1 = game_state.split(';')
		winner = int(grid1[-1])
		grid2 = grid1[:self.rows]
		for y in range(self.rows):
			for x in range(self.cols):
				self.matrix[y][x].add_game(grid2[y][x], winner)

	def get_data(self, y, x, token, winner):
		return self.matrix[y][x].get_prob(token, winner)

	def recorder_heuristic(self, game_state):
		value = 0
		grid1 = game_state.split(';')
		winner = int(grid1[-1])
		grid2 = grid1[:self.rows]
		prob_x = 100.0
		prob_o = 100.0
		prob_d = 100.0

		#check for a winner
		if winner == 1:
			return UPPER_BOUND
		elif winner == 2:
			return LOWER_BOUND
		elif winner == 0:
			return 0

		#calculate some stuff
		for y in range(self.rows):
			for x in range(self.cols):
				prob_x *= self.get_data(y,x,grid2[y][x],1)
				prob_o *= self.get_data(y,x,grid2[y][x],2)
				prob_d *= self.get_data(y,x,grid2[y][x],0)
				while min(prob_x, prob_o, prob_d) < 10:
					prob_x *= 10
					prob_o *= 10
					prob_d *= 10

		#scale
		while max(prob_x, prob_o, prob_d) > 100:
			prob_x /= 10
			prob_o /= 10
			prob_d /= 10

		if prob_d > prob_x and prob_d > prob_o:
			value = 0
		else:
			value = int(prob_x - prob_o)

		#respect the bounds
		if value >= UPPER_BOUND:
			value = UPPER_BOUND-1
		elif value <= LOWER_BOUND:
			value = LOWER_BOUND+1

		return value

<commit_msg>minor text fixes<commit_after>#!/usr/bin/env python
#game.py
import PythonLibraries.matrix_lib as matrix_lib
from alphabeta import UPPER_BOUND
from alphabeta import LOWER_BOUND

class data_square(object):
	def __init__(self, tokens):
		self.data = {}
		self.total = 0
		for t in tokens:
			d = {0:0, 1:0, 2:0}
			self.data[t] = d

	def get_prob(self, token, winner):
		num = 1
		if self.data[token][winner] != 0:
			num = self.data[token][winner]
		return num / float(self.total)

	def get_total(self):
		return self.total

	def get_total_by_token(self,token):
		return sum(self.data[token].values())

	def get_total_by_winner(self, winner):
		return sum([x[winner] for x in self.data.values()])

	def add_game(self, token, winner):
		self.data[token][winner] += 1
		self.total += 1

class Recorder(object):
	def __init__(self, dataFilename, num_rows, num_cols, tokens):
		self.filename = dataFilename
		self.tokens = tokens
		self.rows = num_rows
		self.cols = num_cols
		self.matrix = []
		for y in range(self.rows):
			l = []
			for x in range(self.cols):
				l.append(data_square(self.tokens))
			self.matrix.append(l)
		f = open(self.filename, 'r')
		for line in f:
			self.parse_game(line)
		f.close()

	def parse_game(self, game_state):
		grid1 = game_state.split(';')
		winner = int(grid1[-1])
		grid2 = grid1[:self.rows]
		for y in range(self.rows):
			for x in range(self.cols):
				self.matrix[y][x].add_game(grid2[y][x], winner)

	def get_data(self, y, x, token, winner):
		return self.matrix[y][x].get_prob(token, winner)

	def recorder_heuristic(self, game_state):
		value = 0
		grid1 = game_state.split(';')
		winner = int(grid1[-1])
		grid2 = grid1[:self.rows]
		prob_x = 100.0
		prob_o = 100.0
		prob_d = 100.0

		#check for a winner
		if winner == 1:
			return UPPER_BOUND
		elif winner == 2:
			return LOWER_BOUND
		elif winner == 0:
			return 0

		#calculate some stuff
		for y in range(self.rows):
			for x in range(self.cols):
				prob_x *= self.get_data(y,x,grid2[y][x],1)
				prob_o *= self.get_data(y,x,grid2[y][x],2)
				prob_d *= self.get_data(y,x,grid2[y][x],0)
				while min(prob_x, prob_o, prob_d) < 10:
					prob_x *= 10
					prob_o *= 10
					prob_d *= 10

		#scale
		while max(prob_x, prob_o, prob_d) > 100:
			prob_x /= 10
			prob_o /= 10
			prob_d /= 10

		if prob_d > prob_x and prob_d > prob_o:
			value = 0
		else:
			value = int(prob_x - prob_o)

		#respect the bounds
		if value >= UPPER_BOUND:
			value = UPPER_BOUND-1
		elif value <= LOWER_BOUND:
			value = LOWER_BOUND+1

		return value

<|endoftext|>