<commit_before># -*- coding: utf-8 -*-
"""
Created on Sun Dec 18 13:07:32 2016

@author: tih
"""
import gzip
import zipfile
import gdal
import osr
import os
import pandas as pd
import numpy as np
import netCDF4

def Convert_nc_to_tiff(input_nc, output_folder):
    """
    This function converts the nc file into tiff files

    Keyword Arguments:
    input_nc -- name, name of the adf file
    output_folder -- Name of the output tiff file
    """
    from datetime import date
    import wa.General.raster_conversions as RC

    #All_Data = RC.Open_nc_array(input_nc)

    if type(input_nc) == str:
        nc = netCDF4.Dataset(input_nc)
    elif type(input_nc) == list:
        nc = netCDF4.MFDataset(input_nc)

    Var = nc.variables.keys()[-1]
    All_Data = nc[Var]

    geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(input_nc)

    if epsg == 4326:
        epsg = 'WGS84'

    # Create output folder if needed
    if not os.path.exists(output_folder):
        os.mkdir(output_folder)

    for i in range(0,size_Z):
        if not Time is -9999:
            time_one = Time[i]
            d = date.fromordinal(time_one)
            name = os.path.splitext(os.path.basename(input_nc))[0]
            nameparts = name.split('_')[0:-2]
            name_out = os.path.join(output_folder, '_'.join(nameparts) + '_%d.%02d.%02d.tif' %(d.year, d.month, d.day))
        else:
            name=os.path.splitext(os.path.basename(input_nc))[0]
            name_out = os.path.join(output_folder, name + '.tif')

        Data_one = All_Data[i,:,:]
        Save_as_tiff(name_out, Data_one, geo_out, epsg)

    return()

def Convert_grb2_to_nc(input_wgrib, output_nc, band):

    import wa.General.raster_conversions as RC

    # Get environmental variable
    WA_env_paths = os.environ["WA_PATHS"].split(';')
    GDAL_env_path = WA_env_paths[0]
    GDAL_TRANSLATE_PATH = os.path.join(GDAL_env_path, 'gdal_translate.exe')

    # Create command
    fullCmd = ' '.join(['"%s" -of netcdf -b %d' %(GDAL_TRANSLATE_PATH, band), input_wgrib, output_nc])  # -r {nearest}

    RC.Run_command_window(fullCmd)

    return()

def Convert_adf_to_tiff(input_adf, output_tiff):
    """
    This function converts the adf files into tiff files

    Keyword Arguments:
    input_adf -- name, name of the adf file
    output_tiff -- Name of the output tiff file
    """
    import wa.General.raster_conversions as RC

    # Get environmental variable
    WA_env_paths = os.environ["WA_PATHS"].split(';')
    GDAL_env_path = WA_env_paths[0]
    GDAL_TRANSLATE_PATH = os.path.join(GDAL_env_path, 'gdal_translate.exe')

    # convert data from ESRI GRID to GeoTIFF
    fullCmd = ('"%s" -co COMPRESS=DEFLATE -co PREDICTOR=1 -co '
                   'ZLEVEL=1 -of GTiff %s %s') % (GDAL_TRANSLATE_PATH, input_adf, output_tiff)

    RC.Run_command_window(fullCmd)

    return(output_tiff)

def Extract_Data(input_file, output_folder):
    """
    This function extract the zip files

    Keyword Arguments:
    output_file -- name, name of the file that must be unzipped
    output_folder -- Dir, directory where the unzipped data must be
                           stored
    """
    # extract the data
    z = zipfile.ZipFile(input_file, 'r')
    z.extractall(output_folder)
    z.close()

def Extract_Data_gz(zip_filename, outfilename):
    """
    This function extract the zip files

    Keyword Arguments:
    zip_filename -- name, name of the file that must be unzipped
    outfilename -- Dir, directory where the unzipped data must be
                           stored
    """

    with gzip.GzipFile(zip_filename, 'rb') as zf:
        file_content = zf.read()
        save_file_content = file(outfilename, 'wb')
        save_file_content.write(file_content)
    save_file_content.close()
    zf.close()
    os.remove(zip_filename)

def Save_as_tiff(name='', data='', geo='', projection=''):
    """
    This function save the array as a geotiff

    Keyword arguments:
    name -- string, directory name
    data -- [array], dataset of the geotiff
    geo -- [minimum lon, pixelsize, rotation, maximum lat, rotation,
            pixelsize], (geospatial dataset)
    projection -- integer, the EPSG code
    """
    # save as a geotiff
    driver = gdal.GetDriverByName("GTiff")
    dst_ds = driver.Create(name, int(data.shape[1]), int(data.shape[0]), 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    srse = osr.SpatialReference()
    if projection == '':
        srse.SetWellKnownGeogCS("WGS84")

    else:
        try:
            if not srse.SetWellKnownGeogCS(projection) == 6:
                srse.SetWellKnownGeogCS(projection)
            else:
                try:
                    srse.ImportFromEPSG(int(projection))
                except:
                    srse.ImportFromWkt(projection)
        except:
            try:
                srse.ImportFromEPSG(int(projection))
            except:
                srse.ImportFromWkt(projection)

    dst_ds.SetProjection(srse.ExportToWkt())
    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo)
    dst_ds.GetRasterBand(1).WriteArray(data)
    dst_ds = None
    return()

def Save_as_MEM(data='', geo='', projection=''):
    """
    This function save the array as a memory file

    Keyword arguments:
    data -- [array], dataset of the geotiff
    geo -- [minimum lon, pixelsize, rotation, maximum lat, rotation,
            pixelsize], (geospatial dataset)
    projection -- interger, the EPSG code
    """
    # save as a geotiff
    driver = gdal.GetDriverByName("MEM")
    dst_ds = driver.Create('', int(data.shape[1]), int(data.shape[0]), 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    srse = osr.SpatialReference()
    if projection == '':
        srse.SetWellKnownGeogCS("WGS84")
    else:
        srse.SetWellKnownGeogCS(projection)
    dst_ds.SetProjection(srse.ExportToWkt())
    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo)
    dst_ds.GetRasterBand(1).WriteArray(data)
    return(dst_ds)

def Save_as_NC(namenc, DataCube, Var, Reference_filename,  Startdate = '', Enddate = '', Time_steps = '', Scaling_factor = 1):
    """
    This function save the array as a netcdf file

    Keyword arguments:
    namenc -- string, complete path of the output file with .nc extension
    DataCube -- [array], dataset of the nc file, can be a 2D or 3D array [time, lat, lon], must be same size as reference data
    Var -- string, the name of the variable
    Reference_filename -- string, complete path to the reference file name
    Startdate -- 'YYYY-mm-dd', needs to be filled when you want to save a 3D array,  defines the Start datum of the dataset
    Enddate -- 'YYYY-mm-dd', needs to be filled when you want to save a 3D array, defines the End datum of the dataset
    Time_steps -- 'monthly' or 'daily', needs to be filled when you want to save a 3D array, defines the timestep of the dataset
    Scaling_factor -- number, scaling_factor of the dataset, default = 1
    """
    # Import modules
    import wa.General.raster_conversions as RC
    from netCDF4 import Dataset

    if not os.path.exists(namenc):

        # Get raster information
        geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_filename)

        # Create the lat/lon rasters
        lon = np.arange(size_X)*geo_out[1]+geo_out[0] + 0.5 * geo_out[1]
        lat = np.arange(size_Y)*geo_out[5]+geo_out[3] + 0.5 * geo_out[5]

        # Create the nc file
        nco = Dataset(namenc, 'w', format='NETCDF4_CLASSIC')
        nco.description = '%s data' %Var

        # Create dimensions, variables and attributes:
        nco.createDimension('longitude', size_X)
        nco.createDimension('latitude', size_Y)

        # Create time dimension if the parameter is time dependent
        if Startdate is not '':
            if Time_steps == 'monthly':
                Dates = pd.date_range(Startdate,Enddate,freq = 'MS')
            if Time_steps == 'daily':
                Dates = pd.date_range(Startdate,Enddate,freq = 'D')
            time_or=np.zeros(len(Dates))
            i = 0
            for Date in Dates:
                time_or[i] = Date.toordinal()
                i += 1
            nco.createDimension('time', None)
            timeo = nco.createVariable('time', 'f4', ('time',))
            timeo.units = '%s' %Time_steps
            timeo.standard_name = 'time'

        # Create the lon variable
        lono = nco.createVariable('longitude', 'f4', ('longitude',))
        lono.standard_name = 'longitude'
        lono.units = 'degrees_east'

        # Create the lat variable
        lato = nco.createVariable('latitude', 'f4', ('latitude',))
        lato.standard_name = 'latitude'
        lato.units = 'degrees_north'

        # Create container variable for CRS: lon/lat WGS84 datum
        crso = nco.createVariable('crs', 'i4')
        crso.long_name = 'Lon/Lat Coords in WGS84'
        crso.grid_mapping_name = 'latitude_longitude'
        crso.projection = proj
        crso.longitude_of_prime_meridian = 0.0
        crso.semi_major_axis = 6378137.0
        crso.inverse_flattening = 298.257223563
        crso.geo_reference = geo_out

        # Create the data variable
        if Startdate is not '':
            preco = nco.createVariable('%s' %Var, 'f8',  ('time', 'latitude', 'longitude'), zlib=True, least_significant_digit=1)
            timeo[:]=time_or
        else:
            preco = nco.createVariable('%s' %Var, 'f8',  ('latitude', 'longitude'), zlib=True, least_significant_digit=1)

        # Set the data variable information
        preco.scale_factor = Scaling_factor
        preco.add_offset = 0.00
        preco.grid_mapping = 'crs'
        preco.set_auto_maskandscale(False)

        # Set the lat/lon variable
        lono[:] = lon
        lato[:] = lat

        # Set the data variable
        if Startdate is not '':
            for i in range(len(Dates)):
                preco[i,:,:] = DataCube[i,:,:]*1./np.float(Scaling_factor)
        else:
            preco[:,:] = DataCube[:,:] * 1./np.float(Scaling_factor)

        nco.close()
    return()

def Create_NC_name(Var, Simulation, Dir_Basin, sheet_nmbr, info = ''):

    # Create the output name
    nameOut=''.join(['_'.join([Var,'Simulation%d' % Simulation,'_'.join(info)]),'.nc'])
    namePath = os.path.join(Dir_Basin,'Simulations','Simulation_%d' %Simulation, 'Sheet_%d' %sheet_nmbr)
    if not os.path.exists(namePath):
        os.makedirs(namePath)
    nameTot=os.path.join(namePath,nameOut)

    return(nameTot)

def Convert_dict_to_array(River_dict, Array_dict, Reference_data):

    import numpy as np

    import wa.General.raster_conversions as RC

    # Get raster information
    geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_data)

    # Create ID Matrix
    y,x = np.indices((size_Y, size_X))
    ID_Matrix = np.int32(np.ravel_multi_index(np.vstack((y.ravel(),x.ravel())),(size_Y,size_X),mode='clip').reshape(x.shape)) + 1

    # Get tiff array time dimension:
    time_dimension = int(np.shape(Array_dict[0])[0])

    # create an empty array
    DataCube = np.zeros([time_dimension, size_Y, size_X])

    for river_part in range(0,len(River_dict)):
        for river_pixel in range(1,len(River_dict[river_part])):
            river_pixel_ID = River_dict[river_part][river_pixel]
            if len(np.argwhere(ID_Matrix == river_pixel_ID))>0:
                row, col = np.argwhere(ID_Matrix == river_pixel_ID)[0][:]
                DataCube[:,row,col] = Array_dict[river_part][:,river_pixel]

    return(DataCube)

    <commit_msg>Data conversion problem for time-invariant nc-files.<commit_after># -*- coding: utf-8 -*-
"""
Created on Sun Dec 18 13:07:32 2016

@author: tih
"""
import gzip
import zipfile
import gdal
import osr
import os
import pandas as pd
import numpy as np
import netCDF4

def Convert_nc_to_tiff(input_nc, output_folder):
    """
    This function converts the nc file into tiff files

    Keyword Arguments:
    input_nc -- name, name of the adf file
    output_folder -- Name of the output tiff file
    """
    from datetime import date
    import wa.General.raster_conversions as RC

    #All_Data = RC.Open_nc_array(input_nc)

    if type(input_nc) == str:
        nc = netCDF4.Dataset(input_nc)
    elif type(input_nc) == list:
        nc = netCDF4.MFDataset(input_nc)

    Var = nc.variables.keys()[-1]
    All_Data = nc[Var]

    geo_out, epsg, size_X, size_Y, size_Z, Time = RC.Open_nc_info(input_nc)

    if epsg == 4326:
        epsg = 'WGS84'

    # Create output folder if needed
    if not os.path.exists(output_folder):
        os.mkdir(output_folder)

    for i in range(0,size_Z):
        if not Time == -9999:
            time_one = Time[i]
            d = date.fromordinal(time_one)
            name = os.path.splitext(os.path.basename(input_nc))[0]
            nameparts = name.split('_')[0:-2]
            name_out = os.path.join(output_folder, '_'.join(nameparts) + '_%d.%02d.%02d.tif' %(d.year, d.month, d.day))
            Data_one = All_Data[i,:,:]
        else:
            name=os.path.splitext(os.path.basename(input_nc))[0]
            name_out = os.path.join(output_folder, name + '.tif')
            Data_one = All_Data[:,:]

        Save_as_tiff(name_out, Data_one, geo_out, epsg)

    return()

def Convert_grb2_to_nc(input_wgrib, output_nc, band):

    import wa.General.raster_conversions as RC

    # Get environmental variable
    WA_env_paths = os.environ["WA_PATHS"].split(';')
    GDAL_env_path = WA_env_paths[0]
    GDAL_TRANSLATE_PATH = os.path.join(GDAL_env_path, 'gdal_translate.exe')

    # Create command
    fullCmd = ' '.join(['"%s" -of netcdf -b %d' %(GDAL_TRANSLATE_PATH, band), input_wgrib, output_nc])  # -r {nearest}

    RC.Run_command_window(fullCmd)

    return()

def Convert_adf_to_tiff(input_adf, output_tiff):
    """
    This function converts the adf files into tiff files

    Keyword Arguments:
    input_adf -- name, name of the adf file
    output_tiff -- Name of the output tiff file
    """
    import wa.General.raster_conversions as RC

    # Get environmental variable
    WA_env_paths = os.environ["WA_PATHS"].split(';')
    GDAL_env_path = WA_env_paths[0]
    GDAL_TRANSLATE_PATH = os.path.join(GDAL_env_path, 'gdal_translate.exe')

    # convert data from ESRI GRID to GeoTIFF
    fullCmd = ('"%s" -co COMPRESS=DEFLATE -co PREDICTOR=1 -co '
                   'ZLEVEL=1 -of GTiff %s %s') % (GDAL_TRANSLATE_PATH, input_adf, output_tiff)

    RC.Run_command_window(fullCmd)

    return(output_tiff)

def Extract_Data(input_file, output_folder):
    """
    This function extract the zip files

    Keyword Arguments:
    output_file -- name, name of the file that must be unzipped
    output_folder -- Dir, directory where the unzipped data must be
                           stored
    """
    # extract the data
    z = zipfile.ZipFile(input_file, 'r')
    z.extractall(output_folder)
    z.close()

def Extract_Data_gz(zip_filename, outfilename):
    """
    This function extract the zip files

    Keyword Arguments:
    zip_filename -- name, name of the file that must be unzipped
    outfilename -- Dir, directory where the unzipped data must be
                           stored
    """

    with gzip.GzipFile(zip_filename, 'rb') as zf:
        file_content = zf.read()
        save_file_content = file(outfilename, 'wb')
        save_file_content.write(file_content)
    save_file_content.close()
    zf.close()
    os.remove(zip_filename)

def Save_as_tiff(name='', data='', geo='', projection=''):
    """
    This function save the array as a geotiff

    Keyword arguments:
    name -- string, directory name
    data -- [array], dataset of the geotiff
    geo -- [minimum lon, pixelsize, rotation, maximum lat, rotation,
            pixelsize], (geospatial dataset)
    projection -- integer, the EPSG code
    """
    # save as a geotiff
    driver = gdal.GetDriverByName("GTiff")
    dst_ds = driver.Create(name, int(data.shape[1]), int(data.shape[0]), 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    srse = osr.SpatialReference()
    if projection == '':
        srse.SetWellKnownGeogCS("WGS84")

    else:
        try:
            if not srse.SetWellKnownGeogCS(projection) == 6:
                srse.SetWellKnownGeogCS(projection)
            else:
                try:
                    srse.ImportFromEPSG(int(projection))
                except:
                    srse.ImportFromWkt(projection)
        except:
            try:
                srse.ImportFromEPSG(int(projection))
            except:
                srse.ImportFromWkt(projection)

    dst_ds.SetProjection(srse.ExportToWkt())
    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo)
    dst_ds.GetRasterBand(1).WriteArray(data)
    dst_ds = None
    return()

def Save_as_MEM(data='', geo='', projection=''):
    """
    This function save the array as a memory file

    Keyword arguments:
    data -- [array], dataset of the geotiff
    geo -- [minimum lon, pixelsize, rotation, maximum lat, rotation,
            pixelsize], (geospatial dataset)
    projection -- interger, the EPSG code
    """
    # save as a geotiff
    driver = gdal.GetDriverByName("MEM")
    dst_ds = driver.Create('', int(data.shape[1]), int(data.shape[0]), 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    srse = osr.SpatialReference()
    if projection == '':
        srse.SetWellKnownGeogCS("WGS84")
    else:
        srse.SetWellKnownGeogCS(projection)
    dst_ds.SetProjection(srse.ExportToWkt())
    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo)
    dst_ds.GetRasterBand(1).WriteArray(data)
    return(dst_ds)

def Save_as_NC(namenc, DataCube, Var, Reference_filename,  Startdate = '', Enddate = '', Time_steps = '', Scaling_factor = 1):
    """
    This function save the array as a netcdf file

    Keyword arguments:
    namenc -- string, complete path of the output file with .nc extension
    DataCube -- [array], dataset of the nc file, can be a 2D or 3D array [time, lat, lon], must be same size as reference data
    Var -- string, the name of the variable
    Reference_filename -- string, complete path to the reference file name
    Startdate -- 'YYYY-mm-dd', needs to be filled when you want to save a 3D array,  defines the Start datum of the dataset
    Enddate -- 'YYYY-mm-dd', needs to be filled when you want to save a 3D array, defines the End datum of the dataset
    Time_steps -- 'monthly' or 'daily', needs to be filled when you want to save a 3D array, defines the timestep of the dataset
    Scaling_factor -- number, scaling_factor of the dataset, default = 1
    """
    # Import modules
    import wa.General.raster_conversions as RC
    from netCDF4 import Dataset

    if not os.path.exists(namenc):

        # Get raster information
        geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_filename)

        # Create the lat/lon rasters
        lon = np.arange(size_X)*geo_out[1]+geo_out[0] + 0.5 * geo_out[1]
        lat = np.arange(size_Y)*geo_out[5]+geo_out[3] + 0.5 * geo_out[5]

        # Create the nc file
        nco = Dataset(namenc, 'w', format='NETCDF4_CLASSIC')
        nco.description = '%s data' %Var

        # Create dimensions, variables and attributes:
        nco.createDimension('longitude', size_X)
        nco.createDimension('latitude', size_Y)

        # Create time dimension if the parameter is time dependent
        if Startdate is not '':
            if Time_steps == 'monthly':
                Dates = pd.date_range(Startdate,Enddate,freq = 'MS')
            if Time_steps == 'daily':
                Dates = pd.date_range(Startdate,Enddate,freq = 'D')
            time_or=np.zeros(len(Dates))
            i = 0
            for Date in Dates:
                time_or[i] = Date.toordinal()
                i += 1
            nco.createDimension('time', None)
            timeo = nco.createVariable('time', 'f4', ('time',))
            timeo.units = '%s' %Time_steps
            timeo.standard_name = 'time'

        # Create the lon variable
        lono = nco.createVariable('longitude', 'f4', ('longitude',))
        lono.standard_name = 'longitude'
        lono.units = 'degrees_east'

        # Create the lat variable
        lato = nco.createVariable('latitude', 'f4', ('latitude',))
        lato.standard_name = 'latitude'
        lato.units = 'degrees_north'

        # Create container variable for CRS: lon/lat WGS84 datum
        crso = nco.createVariable('crs', 'i4')
        crso.long_name = 'Lon/Lat Coords in WGS84'
        crso.grid_mapping_name = 'latitude_longitude'
        crso.projection = proj
        crso.longitude_of_prime_meridian = 0.0
        crso.semi_major_axis = 6378137.0
        crso.inverse_flattening = 298.257223563
        crso.geo_reference = geo_out

        # Create the data variable
        if Startdate is not '':
            preco = nco.createVariable('%s' %Var, 'f8',  ('time', 'latitude', 'longitude'), zlib=True, least_significant_digit=1)
            timeo[:]=time_or
        else:
            preco = nco.createVariable('%s' %Var, 'f8',  ('latitude', 'longitude'), zlib=True, least_significant_digit=1)

        # Set the data variable information
        preco.scale_factor = Scaling_factor
        preco.add_offset = 0.00
        preco.grid_mapping = 'crs'
        preco.set_auto_maskandscale(False)

        # Set the lat/lon variable
        lono[:] = lon
        lato[:] = lat

        # Set the data variable
        if Startdate is not '':
            for i in range(len(Dates)):
                preco[i,:,:] = DataCube[i,:,:]*1./np.float(Scaling_factor)
        else:
            preco[:,:] = DataCube[:,:] * 1./np.float(Scaling_factor)

        nco.close()
    return()

def Create_NC_name(Var, Simulation, Dir_Basin, sheet_nmbr, info = ''):

    # Create the output name
    nameOut=''.join(['_'.join([Var,'Simulation%d' % Simulation,'_'.join(info)]),'.nc'])
    namePath = os.path.join(Dir_Basin,'Simulations','Simulation_%d' %Simulation, 'Sheet_%d' %sheet_nmbr)
    if not os.path.exists(namePath):
        os.makedirs(namePath)
    nameTot=os.path.join(namePath,nameOut)

    return(nameTot)

def Convert_dict_to_array(River_dict, Array_dict, Reference_data):

    import numpy as np

    import wa.General.raster_conversions as RC

    # Get raster information
    geo_out, proj, size_X, size_Y = RC.Open_array_info(Reference_data)

    # Create ID Matrix
    y,x = np.indices((size_Y, size_X))
    ID_Matrix = np.int32(np.ravel_multi_index(np.vstack((y.ravel(),x.ravel())),(size_Y,size_X),mode='clip').reshape(x.shape)) + 1

    # Get tiff array time dimension:
    time_dimension = int(np.shape(Array_dict[0])[0])

    # create an empty array
    DataCube = np.zeros([time_dimension, size_Y, size_X])

    for river_part in range(0,len(River_dict)):
        for river_pixel in range(1,len(River_dict[river_part])):
            river_pixel_ID = River_dict[river_part][river_pixel]
            if len(np.argwhere(ID_Matrix == river_pixel_ID))>0:
                row, col = np.argwhere(ID_Matrix == river_pixel_ID)[0][:]
                DataCube[:,row,col] = Array_dict[river_part][:,river_pixel]

    return(DataCube)

    <|endoftext|>


<commit_before>#!/usr/bin/python

from couchdb import Server
import datetime

# server = Server() # connects to the local_server
# >>> remote_server = Server('http://example.com:5984/')
# >>> secure_remote_server = Server('https://username:password@example.com:5984/')

class db_server(object):

    def __init__(self,username,login):
        self.secure_server = Server('http://%s:%s@115.146.95.12:5984' % (username, login))
        self.db = self.secure_server["tweet"]
    def insert(self,data):
        try:
            doc_id,doc_rev=self.db.save(data)
        except Exception as e:
            with open('dabatase_log','a') as f:
                f.write("["+datetime.datetime.now().__str__()+"]"+'\n')
                f.write(str(e)+'\n')
                f.write((data['_id']+'\n'))

<commit_msg>Update connect_db.py<commit_after>#!/usr/bin/python

from couchdb import Server
import datetime
from Tweet import args
# server = Server() # connects to the local_server
# >>> remote_server = Server('http://example.com:5984/')
# >>> secure_remote_server = Server('https://username:password@example.com:5984/')

class db_server(object):

    def __init__(self,username,login):
        self.secure_server = Server('http://%s:%s@115.146.95.12:5984' % (username, login))
        self.db = self.secure_server["tweet"]
    def insert(self,data):
        try:
            doc_id,doc_rev=self.db.save(data)
        except Exception as e:
            with open('dabatase_log_'+args['city'],'a') as f:
                f.write("["+datetime.datetime.now().__str__()+"]"+'\n')
                f.write(str(e)+'\n')
                f.write((data['_id']+'\n'))

<|endoftext|>


<commit_before>""" Helper functions for actions that operate on accounts.

These could be methods of ImapAccount, but separating them gives us more
flexibility with calling code, as most don't need any attributes of the account
object other than the ID, to limit the action.

Types returned for data are the column types defined via SQLAlchemy.

Eventually we're going to want a better way of ACLing functions that operate on
accounts.
"""
from sqlalchemy import func
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound

from inbox.models import Block, Message, SpoolMessage, Folder
from inbox.models.backends.imap import ImapUid, UIDValidity

from inbox.log import get_logger
log = get_logger()

def total_stored_data(account_id, session):
    """
    Computes the total size of the block data of emails in your
    account's IMAP folders.

    """
    subq = session.query(Block) \
        .join(Block.message, Message.imapuid) \
        .filter(ImapUid.imapaccount_id == account_id) \
        .group_by(Message.id, Block.id)
    return session.query(func.sum(subq.subquery().columns.size)).scalar()

def total_stored_messages(account_id, session):
    """ Computes the number of emails in your account's IMAP folders. """
    return session.query(Message) \
        .join(Message.imapuid) \
        .filter(ImapUid.imapaccount_id == account_id) \
        .group_by(Message.id).count()

def num_uids(account_id, session, folder_name):
    return session.query(ImapUid.msg_uid).join(Folder).filter(
        ImapUid.imapaccount_id == account_id,
        Folder.name == folder_name).count()

def all_uids(account_id, session, folder_name):
    return [uid for uid, in session.query(ImapUid.msg_uid).join(Folder).filter(
        ImapUid.imapaccount_id == account_id,
        Folder.name == folder_name)]

def g_msgids(account_id, session, in_=None):
    query = session.query(Message.g_msgid).join(ImapUid) \
        .filter(ImapUid.imapaccount_id == account_id).all()
    # in some cases, in_ can contain +100k items, when the query only
    # returns a few thousand. we shouldn't pass them all to MySQL
    in_ = {long(i) for i in in_}  # in case they are strings
    return sorted([g_msgid for g_msgid, in query if g_msgid in in_])

def g_metadata(account_id, session, folder_name):
    query = session.query(ImapUid.msg_uid, Message.g_msgid, Message.g_thrid)\
        .filter(ImapUid.imapaccount_id == account_id,
                Folder.name == folder_name,
                ImapUid.message_id == Message.id)

    return dict([(uid, dict(msgid=g_msgid, thrid=g_thrid))
                 for uid, g_msgid, g_thrid in query])

def update_thread_labels(thread, folder_name, g_labels, db_session):
    existing_labels = {folder.name.lower() for folder in thread.folders}
    new_labels = {l.lstrip('\\').lower() for l in g_labels}
    new_labels.add(folder_name.lower())

    # Remove labels that have been deleted -- note that the \Inbox, \Sent,
    # \Important, and \Drafts labels are per-message, not per-thread, but
    # since we always work at the thread level, _we_ apply the label to the
    # whole thread.
    thread.folders = {folder for folder in thread.folders if
                      folder.name.lower() in new_labels or
                      folder.name.lower() in ('inbox', 'sent', 'drafts',
                                              'important')}

    # add new labels
    for label in new_labels:
        if label.lower() not in existing_labels:
            # The problem here is that Gmail's attempt to squash labels and
            # IMAP folders into the same abstraction doesn't work
            # perfectly. In particular, there is a '[Gmail]/Sent' folder,
            # but *also* a 'Sent' label, and so on. We handle this by only
            # maintaining one folder object that encapsulates both of
            # these.
            if label == 'Sent':
                thread.folders.add(thread.namespace.account.sent_folder)
            elif label == 'Draft':
                thread.folders.add(thread.namespace.account.drafts_folder)
            elif label == 'Starred':
                thread.folders.add(thread.namespace.account.starred_folder)
            elif label == 'Important':
                thread.folders.add(
                    thread.namespace.account.important_folder)
            else:
                folder = Folder.find_or_create(db_session,
                                               thread.namespace.account,
                                               label)
                thread.folders.add(folder)
    return new_labels

def update_metadata(account_id, session, folder_name, uids, new_flags):
    """ Update flags (the only metadata that can change).

    Make sure you're holding a db write lock on the account. (We don't try
    to grab the lock in here in case the caller needs to put higher-level
    functionality in the lock.)
    """
    for item in session.query(ImapUid).join(Folder)\
            .filter(ImapUid.imapaccount_id == account_id,
                    ImapUid.msg_uid.in_(uids), Folder.name == folder_name)\
            .options(joinedload(ImapUid.message)):
        flags = new_flags[item.msg_uid].flags
        if hasattr(new_flags[item.msg_uid], 'labels'):
            labels = new_flags[item.msg_uid].labels
            thread = item.message.thread
            update_thread_labels(thread, folder_name, labels, session)
        else:
            labels = None
        item.update_imap_flags(flags, labels)
        item.message.is_draft = item.is_draft
        item.message.is_read = item.is_seen

def remove_messages(account_id, session, uids, folder):
    """ Make sure you're holding a db write lock on the account. (We don't try
        to grab the lock in here in case the caller needs to put higher-level
        functionality in the lock.)
    """
    deletes = session.query(ImapUid).join(Folder).filter(
        ImapUid.imapaccount_id == account_id,
        Folder.name == folder,
        ImapUid.msg_uid.in_(uids)).all()

    for d in deletes:
        session.delete(d)
    session.commit()

    # XXX TODO: Have a recurring worker permanently remove dangling
    # messages from the database and block store. (Probably too
    # expensive to do here.)
    # XXX TODO: This doesn't properly update threads to make sure they have
    # the correct folders associated with them, or are deleted when they no
    # longer contain any messages.

def get_uidvalidity(account_id, session, folder_name):
    try:
        # using .one() here may catch duplication bugs
        return session.query(UIDValidity).filter_by(
            imapaccount_id=account_id, folder_name=folder_name).one()
    except NoResultFound:
        return None

def uidvalidity_valid(account_id, session, selected_uidvalidity, folder_name,
                      cached_uidvalidity=None):
    """ Validate UIDVALIDITY on currently selected folder. """
    if cached_uidvalidity is None:
        cached_uidvalidity = get_uidvalidity(account_id,
                                             session, folder_name).uid_validity
        assert type(cached_uidvalidity) == type(selected_uidvalidity), \
            "cached_validity: {0} / selected_uidvalidity: {1}".format(
                type(cached_uidvalidity),
                type(selected_uidvalidity))

    if cached_uidvalidity is None:
        # no row is basically equivalent to UIDVALIDITY == -inf
        return True
    else:
        return selected_uidvalidity >= cached_uidvalidity

def update_uidvalidity(account_id, session, folder_name, uidvalidity,
                       highestmodseq):
    cached_validity = get_uidvalidity(account_id, session, folder_name)
    if cached_validity is None:
        cached_validity = UIDValidity(imapaccount_id=account_id,
                                      folder_name=folder_name)
    cached_validity.highestmodseq = highestmodseq
    cached_validity.uid_validity = uidvalidity
    session.add(cached_validity)

def create_imap_message(db_session, log, account, folder, msg):
    """ IMAP-specific message creation logic.

    This is the one function in this file that gets to take an account
    object instead of an account_id, because we need to relate the
    account to ImapUids for versioning to work, since it needs to look
    up the namespace.

    Returns
    -------
    imapuid : inbox.models.tables.imap.ImapUid
        New db object, which links to new Message and Block objects through
        relationships. All new objects are uncommitted.
    """

    msg_class = SpoolMessage if msg.created else Message
    new_msg = msg_class(account=account, mid=msg.uid, folder_name=folder.name,
                        received_date=msg.internaldate, flags=msg.flags,
                        body_string=msg.body)

    if new_msg:
        imapuid = ImapUid(imapaccount=account, folder=folder,
                          msg_uid=msg.uid, message=new_msg)
        imapuid.update_imap_flags(msg.flags)

        new_msg.is_draft = imapuid.is_draft
        new_msg.is_read = imapuid.is_seen

        # NOTE: This might be a good place to add FolderItem entries for
        # non-Gmail backends.

        return imapuid
<commit_msg>Fix case-sensitivity bug when updating Gmail labels.<commit_after>""" Helper functions for actions that operate on accounts.

These could be methods of ImapAccount, but separating them gives us more
flexibility with calling code, as most don't need any attributes of the account
object other than the ID, to limit the action.

Types returned for data are the column types defined via SQLAlchemy.

Eventually we're going to want a better way of ACLing functions that operate on
accounts.
"""
from sqlalchemy import func
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound

from inbox.models import Block, Message, SpoolMessage, Folder
from inbox.models.backends.imap import ImapUid, UIDValidity

from inbox.log import get_logger
log = get_logger()

def total_stored_data(account_id, session):
    """
    Computes the total size of the block data of emails in your
    account's IMAP folders.

    """
    subq = session.query(Block) \
        .join(Block.message, Message.imapuid) \
        .filter(ImapUid.imapaccount_id == account_id) \
        .group_by(Message.id, Block.id)
    return session.query(func.sum(subq.subquery().columns.size)).scalar()

def total_stored_messages(account_id, session):
    """ Computes the number of emails in your account's IMAP folders. """
    return session.query(Message) \
        .join(Message.imapuid) \
        .filter(ImapUid.imapaccount_id == account_id) \
        .group_by(Message.id).count()

def num_uids(account_id, session, folder_name):
    return session.query(ImapUid.msg_uid).join(Folder).filter(
        ImapUid.imapaccount_id == account_id,
        Folder.name == folder_name).count()

def all_uids(account_id, session, folder_name):
    return [uid for uid, in session.query(ImapUid.msg_uid).join(Folder).filter(
        ImapUid.imapaccount_id == account_id,
        Folder.name == folder_name)]

def g_msgids(account_id, session, in_=None):
    query = session.query(Message.g_msgid).join(ImapUid) \
        .filter(ImapUid.imapaccount_id == account_id).all()
    # in some cases, in_ can contain +100k items, when the query only
    # returns a few thousand. we shouldn't pass them all to MySQL
    in_ = {long(i) for i in in_}  # in case they are strings
    return sorted([g_msgid for g_msgid, in query if g_msgid in in_])

def g_metadata(account_id, session, folder_name):
    query = session.query(ImapUid.msg_uid, Message.g_msgid, Message.g_thrid)\
        .filter(ImapUid.imapaccount_id == account_id,
                Folder.name == folder_name,
                ImapUid.message_id == Message.id)

    return dict([(uid, dict(msgid=g_msgid, thrid=g_thrid))
                 for uid, g_msgid, g_thrid in query])

def update_thread_labels(thread, folder_name, g_labels, db_session):
    existing_labels = {folder.name.lower() for folder in thread.folders}
    new_labels = {l.lstrip('\\').lower() for l in g_labels}
    new_labels.add(folder_name.lower())

    # Remove labels that have been deleted -- note that the \Inbox, \Sent,
    # \Important, and \Drafts labels are per-message, not per-thread, but
    # since we always work at the thread level, _we_ apply the label to the
    # whole thread.
    thread.folders = {folder for folder in thread.folders if
                      folder.name.lower() in new_labels or
                      folder.name.lower() in ('inbox', 'sent', 'drafts',
                                              'important')}

    # add new labels
    for label in new_labels:
        if label.lower() not in existing_labels:
            # The problem here is that Gmail's attempt to squash labels and
            # IMAP folders into the same abstraction doesn't work
            # perfectly. In particular, there is a '[Gmail]/Sent' folder,
            # but *also* a 'Sent' label, and so on. We handle this by only
            # maintaining one folder object that encapsulates both of
            # these.
            if label == 'sent':
                thread.folders.add(thread.namespace.account.sent_folder)
            elif label == 'draft':
                thread.folders.add(thread.namespace.account.drafts_folder)
            elif label == 'starred':
                thread.folders.add(thread.namespace.account.starred_folder)
            elif label == 'important':
                thread.folders.add(
                    thread.namespace.account.important_folder)
            else:
                folder = Folder.find_or_create(db_session,
                                               thread.namespace.account,
                                               label)
                thread.folders.add(folder)
    return new_labels

def update_metadata(account_id, session, folder_name, uids, new_flags):
    """ Update flags (the only metadata that can change).

    Make sure you're holding a db write lock on the account. (We don't try
    to grab the lock in here in case the caller needs to put higher-level
    functionality in the lock.)
    """
    for item in session.query(ImapUid).join(Folder)\
            .filter(ImapUid.imapaccount_id == account_id,
                    ImapUid.msg_uid.in_(uids), Folder.name == folder_name)\
            .options(joinedload(ImapUid.message)):
        flags = new_flags[item.msg_uid].flags
        if hasattr(new_flags[item.msg_uid], 'labels'):
            labels = new_flags[item.msg_uid].labels
            thread = item.message.thread
            update_thread_labels(thread, folder_name, labels, session)
        else:
            labels = None
        item.update_imap_flags(flags, labels)
        item.message.is_draft = item.is_draft
        item.message.is_read = item.is_seen

def remove_messages(account_id, session, uids, folder):
    """ Make sure you're holding a db write lock on the account. (We don't try
        to grab the lock in here in case the caller needs to put higher-level
        functionality in the lock.)
    """
    deletes = session.query(ImapUid).join(Folder).filter(
        ImapUid.imapaccount_id == account_id,
        Folder.name == folder,
        ImapUid.msg_uid.in_(uids)).all()

    for d in deletes:
        session.delete(d)
    session.commit()

    # XXX TODO: Have a recurring worker permanently remove dangling
    # messages from the database and block store. (Probably too
    # expensive to do here.)
    # XXX TODO: This doesn't properly update threads to make sure they have
    # the correct folders associated with them, or are deleted when they no
    # longer contain any messages.

def get_uidvalidity(account_id, session, folder_name):
    try:
        # using .one() here may catch duplication bugs
        return session.query(UIDValidity).filter_by(
            imapaccount_id=account_id, folder_name=folder_name).one()
    except NoResultFound:
        return None

def uidvalidity_valid(account_id, session, selected_uidvalidity, folder_name,
                      cached_uidvalidity=None):
    """ Validate UIDVALIDITY on currently selected folder. """
    if cached_uidvalidity is None:
        cached_uidvalidity = get_uidvalidity(account_id,
                                             session, folder_name).uid_validity
        assert type(cached_uidvalidity) == type(selected_uidvalidity), \
            "cached_validity: {0} / selected_uidvalidity: {1}".format(
                type(cached_uidvalidity),
                type(selected_uidvalidity))

    if cached_uidvalidity is None:
        # no row is basically equivalent to UIDVALIDITY == -inf
        return True
    else:
        return selected_uidvalidity >= cached_uidvalidity

def update_uidvalidity(account_id, session, folder_name, uidvalidity,
                       highestmodseq):
    cached_validity = get_uidvalidity(account_id, session, folder_name)
    if cached_validity is None:
        cached_validity = UIDValidity(imapaccount_id=account_id,
                                      folder_name=folder_name)
    cached_validity.highestmodseq = highestmodseq
    cached_validity.uid_validity = uidvalidity
    session.add(cached_validity)

def create_imap_message(db_session, log, account, folder, msg):
    """ IMAP-specific message creation logic.

    This is the one function in this file that gets to take an account
    object instead of an account_id, because we need to relate the
    account to ImapUids for versioning to work, since it needs to look
    up the namespace.

    Returns
    -------
    imapuid : inbox.models.tables.imap.ImapUid
        New db object, which links to new Message and Block objects through
        relationships. All new objects are uncommitted.
    """

    msg_class = SpoolMessage if msg.created else Message
    new_msg = msg_class(account=account, mid=msg.uid, folder_name=folder.name,
                        received_date=msg.internaldate, flags=msg.flags,
                        body_string=msg.body)

    if new_msg:
        imapuid = ImapUid(imapaccount=account, folder=folder,
                          msg_uid=msg.uid, message=new_msg)
        imapuid.update_imap_flags(msg.flags)

        new_msg.is_draft = imapuid.is_draft
        new_msg.is_read = imapuid.is_seen

        # NOTE: This might be a good place to add FolderItem entries for
        # non-Gmail backends.

        return imapuid
<|endoftext|>


<commit_before>import pickle
import time
import curses
import json
import math
import os.path
import random
import threading

# ideas go here
# lifecycle of a plant
# seed -> seedling -> sprout -> young plant -> mature plant -> flower ->
# pollination -> fruit -> seeds

# mutations over time to yield different things? idk
# neighboring plants can cross pollinate for different plants
# health based on checkups and watering

# development plan
# build plant lifecycle just stepping through
#   What else should it do during life? growth alone is not all that
#   interesting.
#   how long should each stage last ? thinking realistic lmao

# build time system
# build persistence across sessions
# build ascii trees
# build gui?

# build multiplayer

# def display_update:
#     myscreen = curses.initscr()
#
#     myscreen.border(0)
#     myscreen.addstr(1, 2, "you've planted a seed")
#     myscreen.refresh()
#
#     for i in range(1,20):
#         myscreen.addstr(i, 2, str(i))
#         time.sleep(1)
#         myscreen.refresh()
#
#     myscreen.getch()
#
#     curses.endwin()

class Plant:
    def __init__(self):
        self.stage = 0
        self.mutation = 0
        self.species = random.randint(0,5)
        self.color = random.randint(0,10)
        self.rarity = self.rarity_check()

    def rarity_check(self):
        # todo: split this by math not literals
        rare_seed = random.randint(0,127)
        if 0<= rare_seed <= 63:
            rarity = 0
        elif 64 <= rare_seed <= 95:
            rarity = 1
        elif 96 <= rare_seed <= 111:
            rarity = 2
        elif 112 <= rare_seed <= 123:
            rarity = 3
        elif 124 <= rare_seed <= 127:
            rarity = 4
        return rarity

    def growth(self):
        if self.stage < 6:
            self.stage += 1
            # do stage growth stuff
            MUTATION_RARITY = 9
            mutation_seed = random.randint(0,MUTATION_RARITY)
            if mutation_seed == MUTATION_RARITY:
                mutation = random.randint(0,3)
                if self.mutation == 0:
                    self.mutation = mutation
        else:
            # do stage 5 stuff (after fruiting)
            1==1
    def parse_plant(self):
        stage_dict = {
            0: 'seed',
            1: 'seedling',
            2: 'young',
            3: 'grown',
            4: 'mature',
            5: 'flowering',
            6: 'fruiting',
        }

        color_dict = {
            0: 'red',
            1: 'orange',
            2: 'yellow',
            3: 'green',
            4: 'blue',
            5: 'indigo',
            6: 'violet',
            7: 'white',
            8: 'black',
            9: 'gold',
            10: 'rainbow',
        }

        rarity_dict = {
            0: 'common',
            1: 'uncommon',
            2: 'rare',
            3: 'legendary',
            4: 'godly',
        }

        species_dict = {
            0: 'poppy',
            1: 'cactus',
            2: 'aloe',
            3: 'rose',
            4: 'cherry tree',
            5: 'fern',
        }

        mutation_dict = {
            0: '',
            1: 'humming',
            2: 'poisonous',
            3: 'vorpal',
            4: 'glowing'
        }
        if self.mutation == 0:
            print rarity_dict[self.rarity] +" "+ color_dict[self.color] +" "+ stage_dict[self.stage] +" "+ species_dict[self.species]
        else:
            print rarity_dict[self.rarity] +" "+ mutation_dict[self.mutation] +" "+ color_dict[self.color] +" "+ stage_dict[self.stage] +" "+ species_dict[self.species]

if __name__ == '__main__':
    my_plant = Plant()
    print my_plant.stage, my_plant.species, my_plant.color, my_plant.rarity
    while my_plant.stage < 6:
        raw_input("...")
        my_plant.growth()
        my_plant.parse_plant()

    print "end"
<commit_msg>Redo rarity math<commit_after>from __future__ import division
import time
import pickle
import curses
import json
import math
import os.path
import random
import threading

# ideas go here
# lifecycle of a plant
# seed -> seedling -> sprout -> young plant -> mature plant -> flower ->
# pollination -> fruit -> seeds

# mutations over time to yield different things? idk
# neighboring plants can cross pollinate for different plants
# health based on checkups and watering

# development plan
# build plant lifecycle just stepping through
#   What else should it do during life? growth alone is not all that
#   interesting.
#   how long should each stage last ? thinking realistic lmao

# build time system
# build persistence across sessions
# build ascii trees
# build gui?

# build multiplayer

# def display_update:
#     myscreen = curses.initscr()
#
#     myscreen.border(0)
#     myscreen.addstr(1, 2, "you've planted a seed")
#     myscreen.refresh()
#
#     for i in range(1,20):
#         myscreen.addstr(i, 2, str(i))
#         time.sleep(1)
#         myscreen.refresh()
#
#     myscreen.getch()
#
#     curses.endwin()

class Plant:
    def __init__(self):
        self.stage = 0
        self.mutation = 0
        self.species = random.randint(0,5)
        self.color = random.randint(0,10)
        self.rarity = self.rarity_check()

    def rarity_check(self):
        # todo: split this by math not literals
        CONST_RARITY_MAX = 256.0
        rare_seed = random.randint(1,CONST_RARITY_MAX)
        common_range =    round((2/3)*CONST_RARITY_MAX)
        uncommon_range =  round((2/3)*(CONST_RARITY_MAX-common_range))
        rare_range =      round((2/3)*(CONST_RARITY_MAX-common_range-uncommon_range))
        legendary_range = round((2/3)*(CONST_RARITY_MAX-common_range-uncommon_range-rare_range))
        godly_range =     round((2/3)*(CONST_RARITY_MAX-common_range-uncommon_range-rare_range-legendary_range))

        # print common_range, uncommon_range, rare_range, legendary_range, godly_range

        common_max = common_range
        uncommon_max = common_max + uncommon_range
        rare_max = uncommon_max + rare_range
        legendary_max = rare_max + legendary_range
        godly_max = legendary_max + godly_range

        # print common_max, uncommon_max, rare_max, legendary_max, godly_max
        if   0 <= rare_seed <= common_max:
            rarity = 0
        elif common_max < rare_seed <= uncommon_max:
            rarity = 1
        elif uncommon_max < rare_seed <= rare_max:
            rarity = 2
        elif rare_max < rare_seed <= legendary_max:
            rarity = 3
        elif legendary_max < rare_seed <= CONST_RARITY_MAX:
            rarity = 4
        return rarity

    def growth(self):
        if self.stage < 6:
            self.stage += 1
            # do stage growth stuff
            CONST_MUTATION_RARITY = 9
            mutation_seed = random.randint(0,CONST_MUTATION_RARITY)
            if mutation_seed == CONST_MUTATION_RARITY:
                mutation = random.randint(0,3)
                if self.mutation == 0:
                    self.mutation = mutation
        else:
            # do stage 5 stuff (after fruiting)
            1==1
    def parse_plant(self):
        stage_dict = {
            0: 'seed',
            1: 'seedling',
            2: 'young',
            3: 'grown',
            4: 'mature',
            5: 'flowering',
            6: 'fruiting',
        }

        color_dict = {
            0: 'red',
            1: 'orange',
            2: 'yellow',
            3: 'green',
            4: 'blue',
            5: 'indigo',
            6: 'violet',
            7: 'white',
            8: 'black',
            9: 'gold',
            10: 'rainbow',
        }

        rarity_dict = {
            0: 'common',
            1: 'uncommon',
            2: 'rare',
            3: 'legendary',
            4: 'godly',
        }

        species_dict = {
            0: 'poppy',
            1: 'cactus',
            2: 'aloe',
            3: 'rose',
            4: 'cherry tree',
            5: 'fern',
        }

        mutation_dict = {
            0: '',
            1: 'humming',
            2: 'poisonous',
            3: 'vorpal',
            4: 'glowing'
        }
        if self.mutation == 0:
            print rarity_dict[self.rarity] +" "+ color_dict[self.color] +" "+ stage_dict[self.stage] +" "+ species_dict[self.species]
        else:
            print rarity_dict[self.rarity] +" "+ mutation_dict[self.mutation] +" "+ color_dict[self.color] +" "+ stage_dict[self.stage] +" "+ species_dict[self.species]

if __name__ == '__main__':
    my_plant = Plant()
    print my_plant.stage, my_plant.species, my_plant.color, my_plant.rarity
    while my_plant.stage < 6:
        raw_input("...")
        my_plant.growth()
        my_plant.parse_plant()

    print "end"
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""botbot is a bot with customizable behavior."""
import difflib
import re
import sys
import time
import traceback

from multiprocessing import Process

from botbot.bot import Bot
from botbot.conf import (CHANNEL,
                         DEBUG,
                         HOSTNAME,
                         PING_INTERVAL,
                         PORT,
                         REALNAME,
                         RETRY_INTERVAL,
                         SERVER,
                         SERVERNAME,
                         USERNAME)
from botbot.debug import debug
from botbot.definitions import IRC_DEFINITIONS, CUST_DEFINITIONS
from botbot.parser import Parser

B = None

IRC_PARSER = None
CUST_PARSER = None

def connect(host, port, chan):
    """Connect to IRC host, on port, and join chan"""
    debug('Connecting to server {}:{} and joining {}'.format(
            host, port, chan))
    global B
    global IRC_PARSER
    global CUST_PARSER
    B = Bot(host, int(port), debug=DEBUG)
    IRC_PARSER = Parser(B, chan, IRC_DEFINITIONS)
    CUST_PARSER = Parser(B, chan, CUST_DEFINITIONS)
    B.write('NICK {}\r\n'.format(USERNAME))
    B.write('USER {} {} {} :{}\r\n'.format(
                USERNAME, HOSTNAME, SERVERNAME, REALNAME))
    # TODO: Figure out a safe solution for this
    time.sleep(3)
    B.write('JOIN {}\r\n'.format(chan))

def disconnect():
    """Part from IRC channel and disconnect from host"""
    try:
        B.write('DISCONNECT\r\n')
        B.close()
    except:
        e = traceback.format_exc()
        debug(e, log_only=True)

def pinger(host):
    """Ping the IRC server at the given interval until a keyboard interrupt"""
    # TODO: This will fail if the bot ever gets disconnected... we are using
    # multiprocessing not threading.
    while True:
        try:
            time.sleep(PING_INTERVAL)
            B.write('PING {}\r\n'.format(host))
        except KeyboardInterrupt:
            break
        except:
            e = traceback.format_exc()
            debug(e)
            break

def recover(args):
    """Recover from a fatal error"""
    retry = RETRY_INTERVAL
    while True:
        try:
            disconnect()
            time.sleep(retry)
            connect(*args)
            break
        except KeyboardInterrupt:
            break
        except:
            e = traceback.format_exc()
            debug(e)
            retry *= 2

def irc_parse(line):
    """Parse IRC protocol message such as PING, ERROR, NETSPLIT, etc."""
    IRC_PARSER.parse(line)

def cust_parse(line):
    """Parse custom bot commands via PM and channel."""
    CUST_PARSER.parse(line)

def parse(lines):
    """Create a thread for each parser the lines need to run through"""
    for line in lines:
        Process(target=irc_parse, args=(line,)).start()
        Process(target=cust_parse, args=(line,)).start()

if __name__ == '__main__':
    # Connect to the server
    argc = len(sys.argv)
    args = None
    if argc == 4:
        args = (sys.argv[1], sys.argv[2], sys.argv[3])
    elif argc == 3:
        args = (sys.argv[1], sys.argv[2], CHANNEL)
    elif argc == 2:
        if sys.argv[1] in ('-help', '--help'):
            debug('Usage:\n\n$ botbot\n$ botbot <host>\n$ botbot <host>' +
                  ' <port>\n$ botbot <host> <port> <channel>', prefix=False)
            sys.exit(1)
        else:
            args = (sys.argv[1], PORT, CHANNEL)
    elif argc == 1:
        args = (SERVER, PORT, CHANNEL)
    connect(*args)
    debug('welcome')

    # Create a process to ping the server every so often. This allows us to
    # stay connected to the server even if the socket receives no pings. This
    # arises if the client side network is cleaning up tcp connections before
    # the server can ping the client.
    # TODO: This needs to be called again if there's ever an exception besides
    # KeyboardInterrupt.
    Process(target=pinger, args=(SERVER,)).start()

    # Loop until keyboard interrupt
    msg = ''
    while True:
        try:
            msg += B.read()
            if not msg:
                raise Exception('Socket Error: Read returned empty string.')
            lines = msg.split('\r\n')
            msg = lines.pop()
            # Create a new thread for parsing so we can read again
            Process(target=parse, args=(lines,)).start()
        except KeyboardInterrupt:
            disconnect()
            debug('\nGoodbye')
            sys.exit(0)
        except:
            e = traceback.format_exc()
            debug(e)
            debug('\n\nFATAL ERROR\n\nAttempting recovery...')
            recover(args)
<commit_msg>Updated botbot.py docstring.<commit_after>#!/usr/bin/env python
"""
                            _.:` botbot `:._

This is a script which imports the botbot module and starts the botbot client
"""
import difflib
import re
import sys
import time
import traceback

from multiprocessing import Process

from botbot.bot import Bot
from botbot.conf import (CHANNEL,
                         DEBUG,
                         HOSTNAME,
                         PING_INTERVAL,
                         PORT,
                         REALNAME,
                         RETRY_INTERVAL,
                         SERVER,
                         SERVERNAME,
                         USERNAME)
from botbot.debug import debug
from botbot.definitions import IRC_DEFINITIONS, CUST_DEFINITIONS
from botbot.parser import Parser

B = None

IRC_PARSER = None
CUST_PARSER = None

def connect(host, port, chan):
    """Connect to IRC host, on port, and join chan"""
    debug('Connecting to server {}:{} and joining {}'.format(
            host, port, chan))
    global B
    global IRC_PARSER
    global CUST_PARSER
    B = Bot(host, int(port), debug=DEBUG)
    IRC_PARSER = Parser(B, chan, IRC_DEFINITIONS)
    CUST_PARSER = Parser(B, chan, CUST_DEFINITIONS)
    B.write('NICK {}\r\n'.format(USERNAME))
    B.write('USER {} {} {} :{}\r\n'.format(
                USERNAME, HOSTNAME, SERVERNAME, REALNAME))
    # TODO: Figure out a safe solution for this
    time.sleep(3)
    B.write('JOIN {}\r\n'.format(chan))

def disconnect():
    """Part from IRC channel and disconnect from host"""
    try:
        B.write('DISCONNECT\r\n')
        B.close()
    except:
        e = traceback.format_exc()
        debug(e, log_only=True)

def pinger(host):
    """Ping the IRC server at the given interval until a keyboard interrupt"""
    # TODO: This will fail if the bot ever gets disconnected... we are using
    # multiprocessing not threading.
    while True:
        try:
            time.sleep(PING_INTERVAL)
            B.write('PING {}\r\n'.format(host))
        except KeyboardInterrupt:
            break
        except:
            e = traceback.format_exc()
            debug(e)
            break

def recover(args):
    """Recover from a fatal error"""
    retry = RETRY_INTERVAL
    while True:
        try:
            disconnect()
            time.sleep(retry)
            connect(*args)
            break
        except KeyboardInterrupt:
            break
        except:
            e = traceback.format_exc()
            debug(e)
            retry *= 2

def irc_parse(line):
    """Parse IRC protocol message such as PING, ERROR, NETSPLIT, etc."""
    IRC_PARSER.parse(line)

def cust_parse(line):
    """Parse custom bot commands via PM and channel."""
    CUST_PARSER.parse(line)

def parse(lines):
    """Create a thread for each parser the lines need to run through"""
    for line in lines:
        Process(target=irc_parse, args=(line,)).start()
        Process(target=cust_parse, args=(line,)).start()

if __name__ == '__main__':
    # Connect to the server
    argc = len(sys.argv)
    args = None
    if argc == 4:
        args = (sys.argv[1], sys.argv[2], sys.argv[3])
    elif argc == 3:
        args = (sys.argv[1], sys.argv[2], CHANNEL)
    elif argc == 2:
        if sys.argv[1] in ('-help', '--help'):
            debug('Usage:\n\n$ botbot\n$ botbot <host>\n$ botbot <host>' +
                  ' <port>\n$ botbot <host> <port> <channel>', prefix=False)
            sys.exit(1)
        else:
            args = (sys.argv[1], PORT, CHANNEL)
    elif argc == 1:
        args = (SERVER, PORT, CHANNEL)
    connect(*args)
    debug('welcome')

    # Create a process to ping the server every so often. This allows us to
    # stay connected to the server even if the socket receives no pings. This
    # arises if the client side network is cleaning up tcp connections before
    # the server can ping the client.
    # TODO: This needs to be called again if there's ever an exception besides
    # KeyboardInterrupt.
    Process(target=pinger, args=(SERVER,)).start()

    # Loop until keyboard interrupt
    msg = ''
    while True:
        try:
            msg += B.read()
            if not msg:
                raise Exception('Socket Error: Read returned empty string.')
            lines = msg.split('\r\n')
            msg = lines.pop()
            # Create a new thread for parsing so we can read again
            Process(target=parse, args=(lines,)).start()
        except KeyboardInterrupt:
            disconnect()
            debug('\nGoodbye')
            sys.exit(0)
        except:
            e = traceback.format_exc()
            debug(e)
            debug('\n\nFATAL ERROR\n\nAttempting recovery...')
            recover(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# botler: the CCoWMU IRC bot.

import socket
import logging
import sys
import glob

HOST = 'localhost'
PORT = 6667
NICK = 'botler'
IDENT = 'botler'
REALNAME = 'botler'
LEADER = '!'
START_CHANNELS = ['#hackathon']

log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler(sys.stderr))
log.setLevel(logging.DEBUG)

commands = dict()

def command(name, **options):
    '''Decorator for command functions.

    Example usage:

    @command("echo", option="value", ...)
    def echo(nick, channel, message):
        say(channel, '{}: {}'.fomat(nick, message))

    '''
    def decorator(f):
        options['method'] = f
        commands[name] = options
        return f
    return decorator

def send(message):
    # TODO: break message up based on maximum message size
    log.info('[>] {}'.format(message))
    s.send('{}\r\n'.format(message).encode('utf-8'))

def recv():
    # TODO: buffer until one line
    message = s.recv(2048).decode('utf-8')
    log.info('[<] {}'.format(message))
    return message

def say(channel, message):
    send('PRIVMSG {} :{}'.format(channel, message))

def reload_commands():
    '''Reload all source files in commands directory.'''
    global commands
    commands = dict()
    # Globals dict for the command source.
    command_globals = dict(
        command=command,
        say=say,
    )
    for source in glob.glob('commands/*.py'):
        exec(compile(open(source).read(), source, 'exec'), command_globals)

s = socket.socket()
log.info('Connecting to {}:{} as {}'.format(HOST, PORT, NICK))

s.connect((HOST, PORT))
send('NICK {}'.format(NICK))
send('USER {} {} bla :{}'.format(IDENT, HOST, REALNAME))
for channel in START_CHANNELS:
    send('JOIN {}'.format(channel))
    send("PRIVMSG {} :botler is now online and running...".format(channel))
log.info('Connected')

reload_commands()

while 1:
    data = recv()
    # TODO: parse message properly
    if data.find('PING') != -1:
        send('PONG {}'.format(data.split()[1]))
    if data.find('PRIVMSG') != -1:
        parts = data.split(sep=' ', maxsplit=3)
        if len(parts) == 4:
            # Raw parts of the original message
            # source PRIVMSG target :message
            real_source = parts[0]
            real_target = parts[2]
            real_message = parts[3]

            # Extract source user nick
            # :nick!user@host
            nick = real_source.split('!')[0][1:]

            # If a user PRIVMSG's us we appear as the target
            if real_target == NICK:
                channel = nick
            else:
                channel = real_target

            # Final parameter (message) is often prefixed with ':'
            if real_message.startswith(':'):
                message = real_message[1:]
            else:
                message = real_message

            # Check if we need to care about this message
            if message.startswith(LEADER):
                parts = message.split(maxsplit=1)
                command_ = parts[0][len(LEADER):]
                # Give a default value for message if none is provided.
                if len(parts) == 2:
                    message = parts[1]
                else:
                    message = ""
                # Invoke associated command or error
                if command_ in commands:
                    commands[command_]['method'](nick, channel, message)
                elif command_ == 'reload':
                    reload_commands()
                else:
                    say(channel, 'unknown command "{}"'.format(command_))

        else:
            log.warn('Invalid PRIVMSG detected with {} != 4 parts'.format(len(parts)))

# vim: ts=4:sw=4:et
<commit_msg>Update botler.py<commit_after>#!/usr/bin/env python3

# botler: the CCoWMU IRC bot.

import socket
import logging
import sys
import glob

HOST = 'localhost'
PORT = 6667
NICK = 'botler'
IDENT = 'botler'
REALNAME = 'botler'
LEADER = '!'
START_CHANNELS = ['#hackathon']

log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler(sys.stderr))
log.setLevel(logging.DEBUG)

def command(name, **options):
    '''Decorator for command functions.

    Example usage:

    @command("echo", option="value", ...)
    def echo(nick, channel, message):
        say(channel, '{}: {}'.fomat(nick, message))

    '''
    def decorator(f):
        options['method'] = f
        commands[name] = options
        return f
    return decorator

def send(message):
    # TODO: break message up based on maximum message size
    log.info('[>] {}'.format(message))
    s.send('{}\r\n'.format(message).encode('utf-8'))

def recv():
    # TODO: buffer until one line
    message = s.recv(2048).decode('utf-8')
    log.info('[<] {}'.format(message))
    return message

def say(channel, message):
    send('PRIVMSG {} :{}'.format(channel, message))

def reload_commands():
    '''Reload all source files in commands directory.'''
    global commands
    commands = dict()
    # Globals dict for the command source.
    command_globals = dict(
        command=command,
        say=say,
    )
    for source in glob.glob('commands/*.py'):
        exec(compile(open(source).read(), source, 'exec'), command_globals)

s = socket.socket()
log.info('Connecting to {}:{} as {}'.format(HOST, PORT, NICK))

s.connect((HOST, PORT))
send('NICK {}'.format(NICK))
send('USER {} {} bla :{}'.format(IDENT, HOST, REALNAME))
for channel in START_CHANNELS:
    send('JOIN {}'.format(channel))
    send("PRIVMSG {} :botler is now online and running...".format(channel))
log.info('Connected')

reload_commands()

while 1:
    data = recv()
    # TODO: parse message properly
    if data.find('PING') != -1:
        send('PONG {}'.format(data.split()[1]))
    if data.find('PRIVMSG') != -1:
        parts = data.split(sep=' ', maxsplit=3)
        if len(parts) == 4:
            # Raw parts of the original message
            # source PRIVMSG target :message
            real_source = parts[0]
            real_target = parts[2]
            real_message = parts[3]

            # Extract source user nick
            # :nick!user@host
            nick = real_source.split('!')[0][1:]

            # If a user PRIVMSG's us we appear as the target
            if real_target == NICK:
                channel = nick
            else:
                channel = real_target

            # Final parameter (message) is often prefixed with ':'
            if real_message.startswith(':'):
                message = real_message[1:]
            else:
                message = real_message

            # Check if we need to care about this message
            if message.startswith(LEADER):
                parts = message.split(maxsplit=1)
                command_ = parts[0][len(LEADER):]
                # Give a default value for message if none is provided.
                if len(parts) == 2:
                    message = parts[1]
                else:
                    message = ""
                # Invoke associated command or error
                if command_ in commands:
                    commands[command_]['method'](nick, channel, message)
                elif command_ == 'reload':
                    reload_commands()
                else:
                    say(channel, 'unknown command "{}"'.format(command_))

        else:
            log.warn('Invalid PRIVMSG detected with {} != 4 parts'.format(len(parts)))

# vim: ts=4:sw=4:et
<|endoftext|>


<commit_before>r"""Convert raw PASCAL dataset to TFRecord for object_detection.

Example usage:
    ./create_pascal_tf_record --data_dir=/home/user/VOCdevkit \
        --folder=merged \
        --output_path=/home/user/pascal.record
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import hashlib
import io
# import logging
import os
import random
# import re

import PIL.Image
import tensorflow as tf

from os import listdir
# from os.path import isdir, isfile, join

from object_detection.utils import dataset_util
# from object_detection.utils import label_map_util

from stylelens_dataset.images import Images
# from stylelens_dataset.objects import Objects

flags = tf.app.flags
flags.DEFINE_string('data_dir', '', 'Root directory to raw PASCAL VOC dataset.')
flags.DEFINE_string('set', 'train', 'Convert training set, validation set or '
                    'merged set.')
flags.DEFINE_string('annotations_dir', 'Annotations',
                    '(Relative) path to annotations directory.')
flags.DEFINE_string('folder', 'merged', 'Desired challenge folder.')
flags.DEFINE_string('output_path', '', 'Path to output TFRecord')
# flags.DEFINE_string('eval_output_path', '', 'Path to eval output TFRecord')
flags.DEFINE_string('label_map_path', 'data/pascal_label_map.pbtxt',
                    'Path to label map proto')
flags.DEFINE_boolean('ignore_difficult_instances', False, 'Whether to ignore '
                     'difficult instances')
FLAGS = flags.FLAGS
SETS = ['train', 'val', 'trainval', 'test']

def dict_to_tf_example(data, image_subdirectory='JPEGImages'):
  """
    Convert XML derived dict to tf.Example proto.

    Notice that this function normalizes the bounding box coordinates provided
    by the raw data.

    Args:
    data: dict holding PASCAL XML fields for a single image (obtained by
            running dataset_util.recursive_parse_xml_to_dict)
    dataset_directory: Path to root directory holding PASCAL dataset
    label_map_dict: A map from string label names to integers ids.
    ignore_difficult_instances: Whether to skip difficult instances in the
                                dataset  (default: False).
    1image_subdirectory: String specifying subdirectory within the
                        PASCAL dataset directory holding the actual image data.

    Returns:
        example: The converted tf.Example.

    Raises:
        ValueError: if the image pointed to by data['filename']
                    is not a valid JPEG
  """

  full_path = os.path.join('/dataset', data['file'])
  with tf.gfile.GFile(full_path, 'rb') as fid:
    encoded_jpg = fid.read()
  encoded_jpg_io = io.BytesIO(encoded_jpg)
  image = PIL.Image.open(encoded_jpg_io)
  if image.format != 'JPEG':
    raise ValueError('Image format not JPEG')
  key = hashlib.sha256(encoded_jpg).hexdigest()

  xmin = []
  ymin = []
  xmax = []
  ymax = []
  poses = []
  classes = []
  classes_text = []

  width = int(data['width'])
  height = int(data['height'])
  xmin.append(float(data['bbox']['x1']) / width)
  ymin.append(float(data['bbox']['y1']) / height)
  xmax.append(float(data['bbox']['x2']) / width)
  ymax.append(float(data['bbox']['y2']) / height)
  classes_text.append(data['category_name'].encode('utf8'))
  classes.append(int(data['category_class']))
  difficult = int(0)
  truncated = int(0)
  poses.append('Frontal'.encode('utf8'))

  example = tf.train.Example(features=tf.train.Features(feature={
      'image/height': dataset_util.int64_feature(height),
      'image/width': dataset_util.int64_feature(width),
      'image/filename': dataset_util.bytes_feature(
          data['file'].encode('utf8')),
      'image/source_id': dataset_util.bytes_feature(
          str(data['_id']).encode('utf8')),
      'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')),
      'image/encoded': dataset_util.bytes_feature(encoded_jpg),
      'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
      'image/object/bbox/xmin': dataset_util.float_list_feature(xmin),
      'image/object/bbox/xmax': dataset_util.float_list_feature(xmax),
      'image/object/bbox/ymin': dataset_util.float_list_feature(ymin),
      'image/object/bbox/ymax': dataset_util.float_list_feature(ymax),
      'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
      'image/object/class/label': dataset_util.int64_list_feature(classes),
      'image/object/difficult': dataset_util.int64_list_feature(difficult),
      'image/object/truncated': dataset_util.int64_list_feature(truncated),
      'image/object/view': dataset_util.bytes_list_feature(poses),
  }))
  return example

def main(_):
  dataset_api = Images()
  writer = tf.python_io.TFRecordWriter(FLAGS.output_path)

  offset = 0
  limit = 50

  while True:
    try:
      res = dataset_api.get_images_by_source("deepfashion", offset=offset,
                                             limit=limit)
      for image in res:
        tf_data = dict_to_tf_example(image)
        writer.write(tf_data.SerializeToString())

      if limit > len(res):
        print("done")
        break
      else:
        offset = offset + len(res)
    except Exception as e:
      print(str(e))

  writer.close()

if __name__ == '__main__':
  tf.app.run()
<commit_msg>make create<commit_after>r"""Convert raw PASCAL dataset to TFRecord for object_detection.

Example usage:
    ./create_pascal_tf_record --data_dir=/home/user/VOCdevkit \
        --folder=merged \
        --output_path=/home/user/pascal.record
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import hashlib
import io
# import logging
import os
import random
# import re

import PIL.Image
import tensorflow as tf

from os import listdir
# from os.path import isdir, isfile, join

from object_detection.utils import dataset_util
# from object_detection.utils import label_map_util

from stylelens_dataset.images import Images
# from stylelens_dataset.objects import Objects

flags = tf.app.flags
flags.DEFINE_string('data_dir', '', 'Root directory to raw PASCAL VOC dataset.')
flags.DEFINE_string('set', 'train', 'Convert training set, validation set or '
                    'merged set.')
flags.DEFINE_string('annotations_dir', 'Annotations',
                    '(Relative) path to annotations directory.')
flags.DEFINE_string('folder', 'merged', 'Desired challenge folder.')
flags.DEFINE_string('train_output_path', '', 'Path to train output TFRecord')
flags.DEFINE_string('eval_output_path', '', 'Path to eval output TFRecord')
flags.DEFINE_string('label_map_path', 'data/pascal_label_map.pbtxt',
                    'Path to label map proto')
flags.DEFINE_boolean('ignore_difficult_instances', False, 'Whether to ignore '
                     'difficult instances')
FLAGS = flags.FLAGS
SETS = ['train', 'val', 'trainval', 'test']

def dict_to_tf_example(data, image_subdirectory='JPEGImages'):
  """
    Convert XML derived dict to tf.Example proto.

    Notice that this function normalizes the bounding box coordinates provided
    by the raw data.

    Args:
    data: dict holding PASCAL XML fields for a single image (obtained by
            running dataset_util.recursive_parse_xml_to_dict)
    dataset_directory: Path to root directory holding PASCAL dataset
    label_map_dict: A map from string label names to integers ids.
    ignore_difficult_instances: Whether to skip difficult instances in the
                                dataset  (default: False).
    1image_subdirectory: String specifying subdirectory within the
                        PASCAL dataset directory holding the actual image data.

    Returns:
        example: The converted tf.Example.

    Raises:
        ValueError: if the image pointed to by data['filename']
                    is not a valid JPEG
  """

  full_path = os.path.join('/dataset', data['file'])
  with tf.gfile.GFile(full_path, 'rb') as fid:
    encoded_jpg = fid.read()
  encoded_jpg_io = io.BytesIO(encoded_jpg)
  image = PIL.Image.open(encoded_jpg_io)
  if image.format != 'JPEG':
    raise ValueError('Image format not JPEG')
  key = hashlib.sha256(encoded_jpg).hexdigest()

  xmin = []
  ymin = []
  xmax = []
  ymax = []
  poses = []
  classes = []
  classes_text = []

  width = int(data['width'])
  height = int(data['height'])
  xmin.append(float(data['bbox']['x1']) / width)
  ymin.append(float(data['bbox']['y1']) / height)
  xmax.append(float(data['bbox']['x2']) / width)
  ymax.append(float(data['bbox']['y2']) / height)
  classes_text.append(data['category_name'].encode('utf8'))
  classes.append(int(data['category_class']))
  difficult = int(0)
  truncated = int(0)
  poses.append('Frontal'.encode('utf8'))

  example = tf.train.Example(features=tf.train.Features(feature={
      'image/height': dataset_util.int64_feature(height),
      'image/width': dataset_util.int64_feature(width),
      'image/filename': dataset_util.bytes_feature(
          data['file'].encode('utf8')),
      'image/source_id': dataset_util.bytes_feature(
          str(data['_id']).encode('utf8')),
      'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')),
      'image/encoded': dataset_util.bytes_feature(encoded_jpg),
      'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
      'image/object/bbox/xmin': dataset_util.float_list_feature(xmin),
      'image/object/bbox/xmax': dataset_util.float_list_feature(xmax),
      'image/object/bbox/ymin': dataset_util.float_list_feature(ymin),
      'image/object/bbox/ymax': dataset_util.float_list_feature(ymax),
      'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
      'image/object/class/label': dataset_util.int64_list_feature(classes),
      'image/object/difficult': dataset_util.int64_list_feature(difficult),
      'image/object/truncated': dataset_util.int64_list_feature(truncated),
      'image/object/view': dataset_util.bytes_list_feature(poses),
  }))
  return example

def read_from_db(dataset_api, category_class):

  offset = 0
  limit = 50
  cut_idx = int(58000*0.8)
  images = []

  while True:
    try:
      res = dataset_api.get_images_by_category_class(category_class, offset=offset, limit=limit)
      images.extend(res)

      if limit > len(res):
        print("done")
        break

      else:
        offset = offset + len(res)
    except Exception as e:
      print(str(e))

  random.shuffle(images)
  train_images = images[0:cut_idx]
  eval_images = images[cut_idx:-1]

  return (train_images, eval_images)

def main(_):
  dataset_api = Images()
  train_writer = tf.python_io.TFRecordWriter(FLAGS.train_output_path)
  eval_writer = tf.python_io.TFRecordWriter(FLAGS.eval_output_path)

  train_images = []
  eval_images = []

  for i in range(1, 4):
    category_class = str(i)

    (category_train_images, category_eval_images) = read_from_db(dataset_api,
                                                                 category_class)
    train_images.extend(category_train_images)
    eval_images.extend(category_eval_images)

  random.shuffle(train_images)
  random.shuffle(eval_images)

  for image in train_images:
    tf_data = dict_to_tf_example(image)
    train_writer.write(tf_data.SerializeToString())

  for image in eval_images:
    tf_data = dict_to_tf_example(image)
    eval_writer.write(tf_data.SerializeToString())

  train_writer.close()
  eval_writer.close()

if __name__ == '__main__':
  tf.app.run()
<|endoftext|>


<commit_before># Skype to IRC bridge

import sys
import Skype4Py
import irclib
import threading

# stubs

def SendToIrc(skypeChat, user, message):
    ircserver = skype2irc.get(skypeChat.Name)
    if ircserver is not None:
        server = ircservers[ircserver[0]]
        server.connection.privmsg(ircserver[1], "%s: %s" % (user, message))
    else:
        print "Unroutable message from %s" % skypeChat

def SendToSkype(ircserver, ircchannel, user, message):
    skypechat = irc2skype.get((ircserver, ircchannel.lower()))
    if skypechat is not None:
        skype.Chat(skypechat).SendMessage("%s: %s" % (user, message))
    else:
        print "Unroutable message from %s/%s" % (ircserver, ircchannel)

# skype

def OnAttach(status):
    print 'API attachment status: ' + skype.Convert.AttachmentStatusToText(status)
    if status == Skype4Py.apiAttachAvailable:
        skype.Attach()

def OnMessageStatus(Message, Status):
    if Status == 'RECEIVED': # or Status == 'SENT':
        SendToIrc(Message.Chat, Message.FromDisplayName, Message.Body)
    else:
        print "Ignoring from skype: %s/%s" % (Message.FromDisplayName, Message.Body)

skype = Skype4Py.Skype(Transport='x11')
skype.OnAttachmentStatus = OnAttach
skype.OnMessageStatus = OnMessageStatus

print 'Connecting to Skype..'
skype.Attach()
print 'connected to skype'

# irc

class DumbIrcClient(irclib.SimpleIRCClient):
    def __init__(self):
        irclib.SimpleIRCClient.__init__(self)

    def on_welcome(self,c,e):
        print "Welcomed!"
        c.join("#WellingtonLunchChat")
    def on_pubmsg(self,c,e):
        user = e.source()
        user = user[0:user.index('!')]
        SendToSkype(c.server,e.target(), user, e.arguments()[0])

    def run(self):
        self.ircobj.process_forever()

chatmaps = []
chatmaps = [("#chris.andreae/$b81041707fc653fb",
             ("irc.sitharus.com", "#WellingtonLunchChat".lower()))]
skype2irc = {}
irc2skype = {}

ircservers = {}

def setupChatmaps(): # WTB: (let (( ... )) )
    for (skype,irc) in chatmaps:
        skype2irc[skype] = irc
        irc2skype[irc] = skype

setupChatmaps()

sitharus = DumbIrcClient()
sitharus.connect("irc.sitharus.com", 6667, "skype")
threading.Thread(None,lambda: sitharus.run()).start()
ircservers["irc.sitharus.com"] = sitharus

print "Unblocked, manual loop"
Cmd = ''
while not Cmd == 'exit':
    Cmd = raw_input('')
<commit_msg>object-orientified it all<commit_after># Skype to IRC bridge

import sys
import Skype4Py
import irclib
import threading

DEBUG_VERBOSE = True
def debug(str):
    if DEBUG_VERBOSE:
        print str

# There is one skype instance, and one IRC instance per client
# To make a bridge, we create and connect an irc instance for the irc channel,
# then call SkypeClient.bridgeChatToIRC(chatName, ircInstance)

# skype

class SkypeClient:
    def __init__(self):
        self.channels = {}
        self.skype = Skype4Py.Skype(Transport='x11')
        self.skype.OnAttachmentStatus = lambda s: self.onSkypeAttach(s)
        self.skype.OnMessageStatus = lambda m, s: self.onSkypeMessageStatus(m, s)

    def connect(self):
        print 'Connecting to Skype...'
        self.skype.Attach()
        print 'Connected to Skype.'

    def onSkypeAttach(self, status):
        debug('API attachment status: ' + self.skype.Convert.AttachmentStatusToText(status))
        if status == Skype4Py.apiAttachAvailable:
            self.skype.Attach()

    def onSkypeMessageStatus(self, Message, Status):
        debug("Status from skype %s/%s Status==%s" % (Message.FromDisplayName, Message.Body, Status))
        if Status == 'RECEIVED': # or Status == 'SENT':
            chatName = Message.Chat.Name
            if(self.channels[chatName] is not None):
                debug("Dispatched to IRCClient")
                self.channels[chatName].sendUserMessage(Message.FromDisplayName, Message.Body)
            else:
                print "Ignored message from unassociated Skype chat %s" % chatName
        else:
            debug("Ignored (type)")

    def bridgeChatToIRC(self, chatName, ircChannel):
        self.channels[chatName] = ircChannel
        ircChannel.setSkypeChat(SkypeClient.SkypeChat(self.skype, chatName))

    # Object to be passed to the IRC instance to encapsulate communicating with a particular skype chat
    class SkypeChat:
        def __init__(self, skype, chatName):
            self.skype = skype
            self.chatName = chatName

        def sendUserMessage(self, user, message):
            debug("Passed message into Skype %s: <%s> %s" % (self.chatName, user, message))
            self.skype.Chat(self.chatName).SendMessage("%s: %s" % (user, message))

# irc.  Currently one connection per channel, a future optimisation
# would be to separate the IRC server connections from the channels

class IrcChannelClient(irclib.SimpleIRCClient):
    def __init__(self, host, channel, nick):
        irclib.SimpleIRCClient.__init__(self)
        self.host = host
        self.channel = channel.lower()
        self.nick = nick

    def on_welcome(self,c,e):
        print "Welcomed to %s, joining %s!" % (self.host, self.channel)
        c.join(self.channel)
        print "Joined!"

    def on_pubmsg(self,c,e):
        user = e.source()
        user = user[0:user.index('!')]
        message = e.arguments()[0]
        debug("Message from IRC: <%s> %s" % (user, message))
        if self.skypeChat is not None:
            debug("Dispatched to SkypeChat")
            self.skypeChat.sendUserMessage(user, message)
        else:
            print "Channel %s not bridged to skype, discarded message" % (self.channel)

    def run(self):
        self.ircobj.process_forever()

    def connectChannel(self):
        self.connect(self.host, 6667, self.nick)
        threading.Thread(None, lambda: self.run()).start()

    def setSkypeChat(self, skypeChat):
        self.skypeChat = skypeChat

    def sendUserMessage(self, user, message):
        if self.connection is not None:
            debug("Passed message to IRC channel: %s <%s> %s" %(self.channel, user, message))
            self.connection.privmsg(self.channel, "%s: %s" % (user, message))
        else:
            print "IRC Client not connected"

# we maintain a one to one mapping of IRC channels to skype chats

skypeInstance = SkypeClient()
skypeInstance.connect()

ircChannel = IrcChannelClient("irc.sitharus.com", "#wellingtonlunchchat", "cskype")
ircChannel.connectChannel()
skypeInstance.bridgeChatToIRC("#chris.andreae/$b81041707fc653fb", ircChannel)

print "Unblocked, manual loop"
Cmd = ''
while not Cmd == 'exit':
    Cmd = raw_input('')
<|endoftext|>


<commit_before>#!/usr/bin/env python

# I used these to generate my first config file. Might be useful in the future

def urlify(s):
    s = re.sub(r"[^\w\s]", '', s)
    s = re.sub(r"\s+", '-', s)
    return s

#for i in all_my_videos:
#    tmpkey = urlify(i['name']).lower()
#    tmptweet = 'How to sign ' + re.sub(r" \(.*\)", r"", i['name']) + ' in #BSL'

## real code starts here

import tweepy
import time
import sys
import os
import random
from configobj import ConfigObj

config = ConfigObj(os.path.expanduser('~/.bslbot'), unrepr=True)

auth = tweepy.OAuthHandler(config['authentication']['consumer_key'], config['authentication']['consumer_secret'])
auth.set_access_token(config['authentication']['access_key'], config['authentication']['access_secret'])
api = tweepy.API(auth)

def printOrTweet(x):
    """Simple function that currently prints, but will tweet."""
    # print x
    print "You have entered the printOrTweet zone."
    api.update_status(x)

def tweetRandomWord():
    # TODO sort it so that 'Random' actually means 'random selection from tweets that haven't been tweeted yet'
    import random
    """This function tweets a pseudo random word in BSL."""
    # Dictionary of all words we can possibly tweet about
    all_words = config['Tweets']['Words']

    # Choose one of the words
    word = all_words[random.choice(all_words.keys())]
    tweet = word['tweet']
    link = word['link']
    printOrTweet(tweet + "\n" + link)

def tweetAbout(category):
    """Function to choose a semi-random tweet from a predefined list.

    This function looks at all the tweets in a category and then chooses one
    that has not been used in a while. Specifically, it chooses tweets which
    have been tweeted the least.

    E.g. If all but 10 tweets in a category have been tweeted once, and 10 have
    never been tweeted, this function will only choose from those 10."""

    # This little code block is bslbot's method for not tweeting the same thing
    # all the time. In fact, bslbot never tweets the same thing twice (once a
    # category is chosen) until he doesn't have a choice.
    lowest_no_of_times_tweeted = 9001
    candidates_for_tweeting = []
    for i in config['tweets'][category].keys():
        # if this tweet has the lowest no_of_times_tweeted so far dump the
        # rest of them and start the list with this one
        if config['tweets'][category][i]['no_of_times_tweeted'] < lowest_no_of_times_tweeted:
            print 'dumping candidates'
            lowest_no_of_times_tweeted = config['tweets'][category][i]['no_of_times_tweeted']
            print 'lowest times is ', lowest_no_of_times_tweeted
            candidates_for_tweeting = [ i ]
        # otherwise if it's equally untweeted, carry on and add this one to
        # the list
        elif config['tweets'][category][i]['no_of_times_tweeted'] == lowest_no_of_times_tweeted:
            candidates_for_tweeting.append(i)
        # else: rejected (do nothing)

    chosen_tweet = random.choice(candidates_for_tweeting)
    tmptweet = config['tweets'][category][chosen_tweet]['tweet']
    try:
        tmptweet = tmptweet + "\n" + config['tweets'][category][chosen_tweet]['link']
    except KeyError:
        print "No link for this one!"

    config['tweets'][category][chosen_tweet]['no_of_times_tweeted'] +=1
    config.write()

    return tmptweet

def _whatShouldITweetAbout():

    """Internal function allowing bslbot to decide what to tweet about.

    Returns string."""

    free_will =  random.random()

    if free_will <0.005:
        return tweetAbout('selfpromotion')
    elif free_will <0.2:
        return tweetAbout('advice')
    else:
        return tweetAbout('words')

def tweet(text=None, delay=0):
    """Function instructing bslbot to tweet.

    If no arguments are given, bslbot decides what to tweet on his own."""

    import time
    time.sleep(delay)

    if text==None:
        printOrTweet(_whatShouldITweetAbout())
    else:
        printOrTweet(text)

if __name__ == "__main__":
    print "Starting bslbot..."
    tweet(delay=random.randint(1,1))
<commit_msg>Added option in config file for maximum delay.<commit_after>#!/usr/bin/env python

# I used these to generate my first config file. Might be useful in the future

def urlify(s):
    s = re.sub(r"[^\w\s]", '', s)
    s = re.sub(r"\s+", '-', s)
    return s

#for i in all_my_videos:
#    tmpkey = urlify(i['name']).lower()
#    tmptweet = 'How to sign ' + re.sub(r" \(.*\)", r"", i['name']) + ' in #BSL'

## real code starts here

import tweepy
import time
import sys
import os
import random
from configobj import ConfigObj

config = ConfigObj(os.path.expanduser('~/.bslbot'), unrepr=True)

auth = tweepy.OAuthHandler(config['authentication']['consumer_key'], config['authentication']['consumer_secret'])
auth.set_access_token(config['authentication']['access_key'], config['authentication']['access_secret'])
api = tweepy.API(auth)

def printOrTweet(x):
    """Simple function that currently prints, but will tweet."""
    # print x
    print "You have entered the printOrTweet zone."
    api.update_status(x)

def tweetRandomWord():
    # TODO sort it so that 'Random' actually means 'random selection from tweets that haven't been tweeted yet'
    import random
    """This function tweets a pseudo random word in BSL."""
    # Dictionary of all words we can possibly tweet about
    all_words = config['Tweets']['Words']

    # Choose one of the words
    word = all_words[random.choice(all_words.keys())]
    tweet = word['tweet']
    link = word['link']
    printOrTweet(tweet + "\n" + link)

def tweetAbout(category):
    """Function to choose a semi-random tweet from a predefined list.

    This function looks at all the tweets in a category and then chooses one
    that has not been used in a while. Specifically, it chooses tweets which
    have been tweeted the least.

    E.g. If all but 10 tweets in a category have been tweeted once, and 10 have
    never been tweeted, this function will only choose from those 10."""

    # This little code block is bslbot's method for not tweeting the same thing
    # all the time. In fact, bslbot never tweets the same thing twice (once a
    # category is chosen) until he doesn't have a choice.
    lowest_no_of_times_tweeted = 9001
    candidates_for_tweeting = []
    for i in config['tweets'][category].keys():
        # if this tweet has the lowest no_of_times_tweeted so far dump the
        # rest of them and start the list with this one
        if config['tweets'][category][i]['no_of_times_tweeted'] < lowest_no_of_times_tweeted:
            print 'dumping candidates'
            lowest_no_of_times_tweeted = config['tweets'][category][i]['no_of_times_tweeted']
            print 'lowest times is ', lowest_no_of_times_tweeted
            candidates_for_tweeting = [ i ]
        # otherwise if it's equally untweeted, carry on and add this one to
        # the list
        elif config['tweets'][category][i]['no_of_times_tweeted'] == lowest_no_of_times_tweeted:
            candidates_for_tweeting.append(i)
        # else: rejected (do nothing)

    chosen_tweet = random.choice(candidates_for_tweeting)
    tmptweet = config['tweets'][category][chosen_tweet]['tweet']
    try:
        tmptweet = tmptweet + "\n" + config['tweets'][category][chosen_tweet]['link']
    except KeyError:
        print "No link for this one!"

    config['tweets'][category][chosen_tweet]['no_of_times_tweeted'] +=1
    config.write()

    return tmptweet

def _whatShouldITweetAbout():

    """Internal function allowing bslbot to decide what to tweet about.

    Returns string."""

    free_will =  random.random()

    if free_will <0.005:
        return tweetAbout('selfpromotion')
    elif free_will <0.2:
        return tweetAbout('advice')
    else:
        return tweetAbout('words')

def tweet(text=None, delay=0):
    """Function instructing bslbot to tweet.

    If no arguments are given, bslbot decides what to tweet on his own."""

    import time
    time.sleep(delay)

    if text==None:
        printOrTweet(_whatShouldITweetAbout())
    else:
        printOrTweet(text)

if __name__ == "__main__":
    print "Starting bslbot..."
    tweet(delay=random.randint(1,config['misc']['max_delay']))
<|endoftext|>