<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Created by follow on 2016/10/28

from flask import Flask
from flaskext.xmlrpc import XMLRPCHandler, Fault
import jenkins
import time
import base64
import commands
# from jinja2.nodes import Output
import os
# import MySQLdb
import re

app = Flask(__name__)
handler = XMLRPCHandler('api')
handler.connect(app, '/api')
j = jenkins.Jenkins("http://127.1:8080", 'rpcuser', '2266bcc74441b07e9c50ba468a620199')
manager_host = '10.1.2.49'

@handler.register
def Hello(name='follow'):
    if not name:
        raise Fault('WTF')
    return "hello %s" % name

@handler.register
def GetInfo():
    data = j.get_info()
    print data
    return data

@handler.register
def GetQueueInfo():
    data = j.get_queue_info()
    print data
    return data

@handler.register
def GetJobs():
    data = j.get_jobs()
    print data
    return data

@handler.register
def GetBuildConsoleOutput(name, number):
    data = j.get_build_console_output(name, number)
    d1 = base64.b64encode(data)
    print d1
    return d1

@handler.register
def GetJobInfo(n):
    data = j.get_job_info(n)
    return str(data)

@handler.register
def BuildJob(n):
    data = j.build_job(n)
    return data

# @handler.register
# def CleanTheMess():
#     db = MySQLdb.connect("10.168.2.125", "svn", "svnpassword", "svntool")
#     cursor = db.cursor()
#     sql = "update scm_projectstatus set status=0,approve_status=0 where status > 0 and approve_status = 3"
#     sql2 = "delete from scm_proj_with_user where projectid=(select projectid from scm_projectstatus where status > 0 and approve_status = 3)"
#     cursor.execute(sql)
#     cursor.execute(sql2)
#     db.commit()
#     db.close()
#     return

@handler.register
def DoCmd(c):
    print "cmd is  %s" % c
    if c == "start":
        command = "su - scm -c /home/scm/apache-tomcat-7.0.39/bin/startup.sh"
        status = [os.system(command), "nothing"]
        con = False
        while not con:
            pidinfo = commands.getstatusoutput('netstat -nlp | grep 8080 | awk \'{print $7}\' | cut -d / -f 1')
            print "pid is:%s" % str(pidinfo[1])
            if not re.match("\d", pidinfo[1]):
                print "sleep"
                time.sleep(1)
            else:
                con = True
    elif c == "update":
        command = "rm -fr /home/scm/apache-tomcat-7.0.39/work/*; rm -fr /home/scm/apache-tomcat-7.0.39/webapps/*; cp -a /root/.jenkins/jobs/Genscript-SCM-QA/workspace/target/scm-test/scm.war /home/scm/apache-tomcat-7.0.39/webapps/scm.war"
        time.sleep(2)
        status = commands.getstatusoutput(command)
    else:
        pid = commands.getstatusoutput('netstat -nlp | grep 8080 | awk \'{print $7}\' | cut -d / -f 1')[1]
        if re.match("\d", pid):
            command = "kill -9 %s" % pid
            status = commands.getstatusoutput(command)
            print "start to kill %s" % pid
            print status
        time.sleep(2)
    return status

@handler.register
def GetProcessInfo():
    status = {}
    pidinfo = commands.getstatusoutput('netstat -nlp | grep 8080 | awk \'{print $7}\' | cut -d / -f 1')
    status['qa_mtime'] = commands.getoutput(
        'stat  /home/scm/apache-tomcat-7.0.39/webapps/scm.war | grep \'^Modify\' | cut  -d " " -f 2-3 | cut -d . -f1')
    status['newest_mtime'] = commands.getoutput(
        'stat  /opt/scm-manager/wars/*.war | grep \'^Modify\' | cut  -d " " -f 2-3 | cut -d . -f1')
    status['load_info'] = commands.getoutput(' w |grep \'load\' | cut -d , -f 4,5,6')
    if pidinfo[1] == "":
        return status
    else:
        status['pid'] = pidinfo[1]
        status['uptime'] = commands.getoutput('ps -p %s -o lstart | sed -n \'2p\'' % pidinfo[1])
        status['mem_info'] = commands.getoutput("cat /proc/{}/status  | grep RSS".format(pidinfo[1]))
        return status

@handler.register
def GetBuildInfo(name, number):
    data = j.get_build_info(name, number)
    t1 = str(data['timestamp'])
    duration = data['duration'] / 1000
    t2 = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(t1[:10])))
    data['timestamp'] = t2
    data['duration'] = duration
    data['changeSet'] = 'changeSet'
    return data  # xml-rpc 不能使用长整数  WTF

@handler.register
def DownloadPackage(path, filename):
    file_url = "http://{}:{}/uploaded_file/{}?folder=SCM-{}".format(manager_host, "80", filename, path)
    download_dir = "/opt/scm-manager/wars"
    download_command = "aria2c -s 2 -x 2 {} -d {} -D".format(file_url, download_dir)
    commands.getoutput("rm -f {}/*.war".format(download_dir))
    output = commands.getoutput(download_command)
    print output
    return output

app.run('0.0.0.0', port=8085, debug=True)
<commit_msg>add war filename<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Created by follow on 2016/10/28

from flask import Flask
from flaskext.xmlrpc import XMLRPCHandler, Fault
import jenkins
import time
import base64
import commands
# from jinja2.nodes import Output
import os
# import MySQLdb
import re

app = Flask(__name__)
handler = XMLRPCHandler('api')
handler.connect(app, '/api')
j = jenkins.Jenkins("http://127.1:8080", 'rpcuser', '2266bcc74441b07e9c50ba468a620199')
manager_host = '10.1.2.49'

@handler.register
def Hello(name='follow'):
    if not name:
        raise Fault('WTF')
    return "hello %s" % name

@handler.register
def GetInfo():
    data = j.get_info()
    print data
    return data

@handler.register
def GetQueueInfo():
    data = j.get_queue_info()
    print data
    return data

@handler.register
def GetJobs():
    data = j.get_jobs()
    print data
    return data

@handler.register
def GetBuildConsoleOutput(name, number):
    data = j.get_build_console_output(name, number)
    d1 = base64.b64encode(data)
    print d1
    return d1

@handler.register
def GetJobInfo(n):
    data = j.get_job_info(n)
    return str(data)

@handler.register
def BuildJob(n):
    data = j.build_job(n)
    return data

# @handler.register
# def CleanTheMess():
#     db = MySQLdb.connect("10.168.2.125", "svn", "svnpassword", "svntool")
#     cursor = db.cursor()
#     sql = "update scm_projectstatus set status=0,approve_status=0 where status > 0 and approve_status = 3"
#     sql2 = "delete from scm_proj_with_user where projectid=(select projectid from scm_projectstatus where status > 0 and approve_status = 3)"
#     cursor.execute(sql)
#     cursor.execute(sql2)
#     db.commit()
#     db.close()
#     return

@handler.register
def DoCmd(c):
    print "cmd is  %s" % c
    if c == "start":
        command = "su - scm -c /home/scm/apache-tomcat-7.0.39/bin/startup.sh"
        status = [os.system(command), "nothing"]
        con = False
        while not con:
            pidinfo = commands.getstatusoutput('netstat -nlp | grep 8080 | awk \'{print $7}\' | cut -d / -f 1')
            print "pid is:%s" % str(pidinfo[1])
            if not re.match("\d", pidinfo[1]):
                print "sleep"
                time.sleep(1)
            else:
                con = True
    elif c == "update":
        command = "rm -fr /home/scm/apache-tomcat-7.0.39/work/*; rm -fr /home/scm/apache-tomcat-7.0.39/webapps/*; cp -a /root/.jenkins/jobs/Genscript-SCM-QA/workspace/target/scm-test/scm.war /home/scm/apache-tomcat-7.0.39/webapps/scm.war"
        time.sleep(2)
        status = commands.getstatusoutput(command)
    else:
        pid = commands.getstatusoutput('netstat -nlp | grep 8080 | awk \'{print $7}\' | cut -d / -f 1')[1]
        if re.match("\d", pid):
            command = "kill -9 %s" % pid
            status = commands.getstatusoutput(command)
            print "start to kill %s" % pid
            print status
        time.sleep(2)
    return status

@handler.register
def GetProcessInfo():
    status = {}
    pidinfo = commands.getstatusoutput('netstat -nlp | grep 8080 | awk \'{print $7}\' | cut -d / -f 1')
    status['qa_mtime'] = commands.getoutput(
        'stat  /home/scm/apache-tomcat-7.0.39/webapps/scm.war | grep \'^Modify\' | cut  -d " " -f 2-3 | cut -d . -f1')
    status['newest_filename'] = commands.getoutput('ls /opt/scm-manager/wars/').lstrip()
    status['newest_mtime'] = commands.getoutput(
        'stat  /opt/scm-manager/wars/*.war | grep \'^Modify\' | cut  -d " " -f 2-3 | cut -d . -f1')
    status['load_info'] = commands.getoutput(' w |grep \'load\' | cut -d , -f 4,5,6')
    if pidinfo[1] == "":
        return status
    else:
        status['pid'] = pidinfo[1]
        status['uptime'] = commands.getoutput('ps -p %s -o lstart | sed -n \'2p\'' % pidinfo[1])
        status['mem_info'] = commands.getoutput("cat /proc/{}/status  | grep RSS".format(pidinfo[1]))
        return status

@handler.register
def GetBuildInfo(name, number):
    data = j.get_build_info(name, number)
    t1 = str(data['timestamp'])
    duration = data['duration'] / 1000
    t2 = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(t1[:10])))
    data['timestamp'] = t2
    data['duration'] = duration
    data['changeSet'] = 'changeSet'
    return data  # xml-rpc 不能使用长整数  WTF

@handler.register
def DownloadPackage(path, filename):
    file_url = "http://{}:{}/uploaded_file/{}?folder=SCM-{}".format(manager_host, "80", filename, path)
    download_dir = "/opt/scm-manager/wars"
    download_command = "aria2c -s 2 -x 2 {} -d {} -D".format(file_url, download_dir)
    commands.getoutput("rm -f {}/*.war".format(download_dir))
    output = commands.getoutput(download_command)
    print output
    return output

app.run('0.0.0.0', port=8085, debug=True)
<|endoftext|>


<commit_before># ndvi.py
#
# Copyright(c) Exequiel Ceasar Navarrete <esnavarrete1@up.edu.ph>
# Licensed under MIT
# Version 1.0.0-alpha6

import ee
import csv
import StringIO
from datetime import datetime
from flask import Blueprint, jsonify, request, abort, make_response
from flask_cors import cross_origin
from app.gzipped import gzipped
from app import EE_CREDENTIALS, cache, app

mod = Blueprint('ndvi', __name__, url_prefix='/ndvi')

def ndvi_mapper(image):
  hansen_image = ee.Image('UMD/hansen/global_forest_change_2013')
  data = hansen_image.select('datamask')
  mask = data.eq(1)

  return image.updateMask(mask)

def time_series_mapper(item):
  prefix = 'MOD13Q1_005_'
  prefixed_date = str(item[0])
  date = ''

  result = prefixed_date.startswith(prefix)

  if result is True:
    date = prefixed_date[len(prefix):].replace('_', '-')

  ndvi_value = float(item[4]) / 10000

  return {
    'time': date,
    'ndvi': ndvi_value
  }

def ndvi_clipper(image):
  ft = "ft:%s" % app.config['PROVINCES_FT']['LOCATION_METADATA_FUSION_TABLE']
  province = ee.FeatureCollection(ft)

  place = request.args.get('place')

  return image.clip(
    province.filter(ee.Filter.eq(app.config['PROVINCES_FT']['LOCATION_FUSION_TABLE_NAME_COLUMN'], place))
    .geometry()
  )

def ndvi_cache_key(*args, **kwargs):
  path = request.path
  args = str(hash(frozenset(request.args.items())))
  return (path + args).encode('utf-8')

def query_time_series_data(lat, lng, start_date, end_date):
  cache_key = 'ndvi_time_series_%s_%s_%s_%s' % (lat, lng, start_date, end_date)

  final_result = cache.get(cache_key)

  if final_result is None:
    ee.Initialize(EE_CREDENTIALS)

    # create a geometry point instance for cropping data later
    point = ee.Geometry.Point(float(lng), float(lat))

    # use the MODIS satellite data (NDVI)
    modis = ee.ImageCollection('MODIS/MOD13Q1').select('NDVI')

    # check first if the resulting date filter yields greater than 0 features
    filtering_result = modis.filterDate(start_date, end_date)

    if len(filtering_result.getInfo()['features']) == 0:
      return None

    result = filtering_result.getRegion(point, 250).getInfo()
    result.pop(0)

    final_result = map(time_series_mapper, result)

    # cache it for 12 hours
    cache.set(cache_key, final_result, timeout=43200)

  return final_result

def query_doy_data(lat, lng, start_date, end_date):
  cache_key = 'ndvi_doy_%s_%s_%s_%s' % (lat, lng, start_date, end_date)

  processed = cache.get(cache_key)

  # perform query processing when its not on the cache
  if processed is None:
    query_result = query_time_series_data(lat, lng, start_date, end_date)

    processed = {}
    for value in query_result:
      parsed_date = datetime.strptime(value['time'], '%Y-%m-%d')
      doy = int(parsed_date.strftime('%j'))
      year = parsed_date.strftime('%Y')

      if processed.get(year) is None:
        processed[year] = {}

      processed[year][doy] = value['ndvi']

    # cache it for 12 hours
    cache.set(cache_key, processed, timeout=43200)

  return processed

# cache the result of this endpoint for 12 hours
@mod.route('/<start_date>/<end_date>', methods=['GET'])
@cross_origin()
@gzipped
@cache.cached(timeout=43200, key_prefix=ndvi_cache_key)
def date_and_range(start_date, end_date):
  ee.Initialize(EE_CREDENTIALS)

  geometric_bounds = ee.List([
    [127.94248139921513, 5.33459854167601],
    [126.74931782819613, 11.825234466620996],
    [124.51107186428203, 17.961503806746318],
    [121.42999903167879, 19.993626604011016],
    [118.25656974884657, 18.2117821750514],
    [116.27168958893185, 6.817365082528201],
    [122.50121143769957, 3.79887124351577],
    [127.94248139921513, 5.33459854167601]
  ])

  geometry = ee.Geometry.Polygon(geometric_bounds, 'EPSG:4326', True)

  image_collection = ee.ImageCollection('LANDSAT/LC8_L1T')
  filtered = image_collection.filterDate(start_date, end_date).filterBounds(geometry).map(ndvi_mapper)

  # clip the image collection to a certain place only
  if request.args.get('place') is not None:
    filtered = filtered.map(ndvi_clipper)

  ndvi = ee.Algorithms.Landsat.simpleComposite(filtered, 50, 10, 40, True).select('B7', 'B4', 'B3')

  map_parameters = {
    'min': 0.05,
    'max': 0.1,
    'gamma': 1.6,
  }

  map_object = ndvi.getMapId(map_parameters)
  map_id = map_object['mapid']
  map_token = map_object['token']

  # assemble the resulting response
  result = {
    'success': True,
    'mapId': map_id,
    'mapToken': map_token
  }

  return jsonify(**result)

@mod.route('/time-series/<lat>/<lng>/<start_date>/<end_date>', methods=['GET'])
@cross_origin()
@gzipped
def time_series(lat, lng, start_date, end_date):
  query_result = query_time_series_data(lat, lng, start_date, end_date)
  output_format = 'json'
  available_formats = ['json', 'csv']
  requested_format = request.args.get('fmt')

  if requested_format is not None:
    # abort the request and throw HTTP 400 since the format
    # is not on the list of available formats
    if not requested_format in available_formats:
      abort(400, 'Unsupported format')

    # override the default output format
    output_format = requested_format

  # abort the request if the query_result contains None value
  if query_result is None:
    abort(404, 'NDVI data not found')

  response = None

  if output_format == 'json':
    json_result = {
      'success': True,
      'result': query_result
    }

    response = jsonify(**json_result)
  else:
    si = StringIO.StringIO()
    cw = csv.writer(si)

    cw.writerow(['Time', 'NDVI'])
    for value in query_result:
      cw.writerow([
        value['time'],
        value['ndvi']
      ])

    filename = 'ndvi-time-series-%s-%s-%s-%s' % (lat, lng, start_date, end_date)

    response = make_response(si.getvalue())
    response.headers['Content-Disposition'] = 'attachment; filename=%s.csv' % filename
    response.headers['Content-type'] = 'text/csv'

  return response

@mod.route('/day-of-the-year/<lat>/<lng>/<start_date>/<end_date>', methods=['GET'])
@cross_origin()
@gzipped
def day_of_the_year(lat, lng, start_date, end_date):
  query_result = query_doy_data(lat, lng, start_date, end_date)
  output_format = 'json'
  available_formats = ['json', 'csv']
  requested_format = request.args.get('fmt')

  if requested_format is not None:
    # abort the request and throw HTTP 400 since the format
    # is not on the list of available formats
    if not requested_format in available_formats:
      abort(400, 'Unsupported format')

    # override the default output format
    output_format = requested_format

  # abort the request if the query_result contains None value
  if query_result is None:
    abort(404, 'NDVI data not found')

  response = None

  if output_format == 'json':
    json_result = {
      'success': True,
      'result': query_result
    }

    response = jsonify(**json_result)
  else:
    si = StringIO.StringIO()
    cw = csv.writer(si)

    csv_header_row = query_result.keys()

    # sort the items
    csv_header_row.sort()

    # save the sorted keys
    years = csv_header_row[:]

    # add the DOY to the first element of the row
    csv_header_row.insert(0, 'Day of the Year')

    # loop through all the doy and years and assemble the csv rows
    cw.writerow(csv_header_row)
    for doy in range(1, 365, 16):
      row = [doy]

      for year in years:
        row.append(query_result[year].get(doy, ''))

      cw.writerow(row)

    filename = 'ndvi-doy-%s-%s-%s-%s' % (lat, lng, start_date, end_date)

    response = make_response(si.getvalue())
    response.headers['Content-Disposition'] = 'attachment; filename=%s.csv' % filename
    response.headers['Content-type'] = 'text/csv'

  return response

<commit_msg>Replace the satellite LANDSAT/LC8_L1T with LANDSAT/LC8_L1T_8DAY_NDVI composite<commit_after># ndvi.py
#
# Copyright(c) Exequiel Ceasar Navarrete <esnavarrete1@up.edu.ph>
# Licensed under MIT
# Version 1.0.0-alpha6

import ee
import csv
import StringIO
from datetime import datetime
from flask import Blueprint, jsonify, request, abort, make_response
from flask_cors import cross_origin
from app.gzipped import gzipped
from app import EE_CREDENTIALS, cache, app

mod = Blueprint('ndvi', __name__, url_prefix='/ndvi')

def ndvi_mapper(image):
  hansen_image = ee.Image('UMD/hansen/global_forest_change_2013')
  data = hansen_image.select('datamask')
  mask = data.eq(1)

  return image.updateMask(mask)

def time_series_mapper(item):
  prefix = 'MOD13Q1_005_'
  prefixed_date = str(item[0])
  date = ''

  result = prefixed_date.startswith(prefix)

  if result is True:
    date = prefixed_date[len(prefix):].replace('_', '-')

  ndvi_value = float(item[4]) / 10000

  return {
    'time': date,
    'ndvi': ndvi_value
  }

def ndvi_clipper(image):
  ft = "ft:%s" % app.config['PROVINCES_FT']['LOCATION_METADATA_FUSION_TABLE']
  province = ee.FeatureCollection(ft)

  place = request.args.get('place')

  return image.clip(
    province.filter(ee.Filter.eq(app.config['PROVINCES_FT']['LOCATION_FUSION_TABLE_NAME_COLUMN'], place))
    .geometry()
  )

def ndvi_cache_key(*args, **kwargs):
  path = request.path
  args = str(hash(frozenset(request.args.items())))
  return (path + args).encode('utf-8')

def query_time_series_data(lat, lng, start_date, end_date):
  cache_key = 'ndvi_time_series_%s_%s_%s_%s' % (lat, lng, start_date, end_date)

  final_result = cache.get(cache_key)

  if final_result is None:
    ee.Initialize(EE_CREDENTIALS)

    # create a geometry point instance for cropping data later
    point = ee.Geometry.Point(float(lng), float(lat))

    # use the MODIS satellite data (NDVI)
    modis = ee.ImageCollection('MODIS/MOD13Q1').select('NDVI')

    # check first if the resulting date filter yields greater than 0 features
    filtering_result = modis.filterDate(start_date, end_date)

    if len(filtering_result.getInfo()['features']) == 0:
      return None

    result = filtering_result.getRegion(point, 250).getInfo()
    result.pop(0)

    final_result = map(time_series_mapper, result)

    # cache it for 12 hours
    cache.set(cache_key, final_result, timeout=43200)

  return final_result

def query_doy_data(lat, lng, start_date, end_date):
  cache_key = 'ndvi_doy_%s_%s_%s_%s' % (lat, lng, start_date, end_date)

  processed = cache.get(cache_key)

  # perform query processing when its not on the cache
  if processed is None:
    query_result = query_time_series_data(lat, lng, start_date, end_date)

    processed = {}
    for value in query_result:
      parsed_date = datetime.strptime(value['time'], '%Y-%m-%d')
      doy = int(parsed_date.strftime('%j'))
      year = parsed_date.strftime('%Y')

      if processed.get(year) is None:
        processed[year] = {}

      processed[year][doy] = value['ndvi']

    # cache it for 12 hours
    cache.set(cache_key, processed, timeout=43200)

  return processed

# cache the result of this endpoint for 12 hours
@mod.route('/<start_date>/<end_date>', methods=['GET'])
@cross_origin()
@gzipped
@cache.cached(timeout=43200, key_prefix=ndvi_cache_key)
def date_and_range(start_date, end_date):
  ee.Initialize(EE_CREDENTIALS)

  geometric_bounds = ee.List([
    [127.94248139921513, 5.33459854167601],
    [126.74931782819613, 11.825234466620996],
    [124.51107186428203, 17.961503806746318],
    [121.42999903167879, 19.993626604011016],
    [118.25656974884657, 18.2117821750514],
    [116.27168958893185, 6.817365082528201],
    [122.50121143769957, 3.79887124351577],
    [127.94248139921513, 5.33459854167601]
  ])

  geometry = ee.Geometry.Polygon(geometric_bounds, 'EPSG:4326', True)

  # performing filterBounds will not work on this since this is a global composite
  # see similar issue with discussion:
  # <https://groups.google.com/forum/#!searchin/google-earth-engine-developers/filterbounds%7Csort:relevance/google-earth-engine-developers/__3vYdhh22k/wIE-INHXBQAJ>
  image_collection = ee.ImageCollection('LANDSAT/LC8_L1T_8DAY_NDVI')
  # filtered = image_collection.filterDate(start_date, end_date).filterBounds(geometry).map(ndvi_mapper)
  filtered = image_collection.filterDate(start_date, end_date).map(ndvi_mapper)

  # WORKAROUND: perform temporal reduction using mean/median reducers to clip it to a certain geometry
  filtered = filtered.mean().clip(geometry)

  # clip the image collection to a certain place only
  if request.args.get('place') is not None:
    filtered = filtered.map(ndvi_clipper)

  ndvi = filtered

  map_parameters = {
    'min': 0,
    'max': 1,
    'palette': 'FFFFFF, CE7E45, FCD163, 66A000, 207401, 056201, 004C00, 023B01, 012E01, 011301'
  }

  map_object = ndvi.getMapId(map_parameters)
  map_id = map_object['mapid']
  map_token = map_object['token']

  # assemble the resulting response
  result = {
    'success': True,
    'mapId': map_id,
    'mapToken': map_token
  }

  return jsonify(**result)

@mod.route('/time-series/<lat>/<lng>/<start_date>/<end_date>', methods=['GET'])
@cross_origin()
@gzipped
def time_series(lat, lng, start_date, end_date):
  query_result = query_time_series_data(lat, lng, start_date, end_date)
  output_format = 'json'
  available_formats = ['json', 'csv']
  requested_format = request.args.get('fmt')

  if requested_format is not None:
    # abort the request and throw HTTP 400 since the format
    # is not on the list of available formats
    if not requested_format in available_formats:
      abort(400, 'Unsupported format')

    # override the default output format
    output_format = requested_format

  # abort the request if the query_result contains None value
  if query_result is None:
    abort(404, 'NDVI data not found')

  response = None

  if output_format == 'json':
    json_result = {
      'success': True,
      'result': query_result
    }

    response = jsonify(**json_result)
  else:
    si = StringIO.StringIO()
    cw = csv.writer(si)

    cw.writerow(['Time', 'NDVI'])
    for value in query_result:
      cw.writerow([
        value['time'],
        value['ndvi']
      ])

    filename = 'ndvi-time-series-%s-%s-%s-%s' % (lat, lng, start_date, end_date)

    response = make_response(si.getvalue())
    response.headers['Content-Disposition'] = 'attachment; filename=%s.csv' % filename
    response.headers['Content-type'] = 'text/csv'

  return response

@mod.route('/day-of-the-year/<lat>/<lng>/<start_date>/<end_date>', methods=['GET'])
@cross_origin()
@gzipped
def day_of_the_year(lat, lng, start_date, end_date):
  query_result = query_doy_data(lat, lng, start_date, end_date)
  output_format = 'json'
  available_formats = ['json', 'csv']
  requested_format = request.args.get('fmt')

  if requested_format is not None:
    # abort the request and throw HTTP 400 since the format
    # is not on the list of available formats
    if not requested_format in available_formats:
      abort(400, 'Unsupported format')

    # override the default output format
    output_format = requested_format

  # abort the request if the query_result contains None value
  if query_result is None:
    abort(404, 'NDVI data not found')

  response = None

  if output_format == 'json':
    json_result = {
      'success': True,
      'result': query_result
    }

    response = jsonify(**json_result)
  else:
    si = StringIO.StringIO()
    cw = csv.writer(si)

    csv_header_row = query_result.keys()

    # sort the items
    csv_header_row.sort()

    # save the sorted keys
    years = csv_header_row[:]

    # add the DOY to the first element of the row
    csv_header_row.insert(0, 'Day of the Year')

    # loop through all the doy and years and assemble the csv rows
    cw.writerow(csv_header_row)
    for doy in range(1, 365, 16):
      row = [doy]

      for year in years:
        row.append(query_result[year].get(doy, ''))

      cw.writerow(row)

    filename = 'ndvi-doy-%s-%s-%s-%s' % (lat, lng, start_date, end_date)

    response = make_response(si.getvalue())
    response.headers['Content-Disposition'] = 'attachment; filename=%s.csv' % filename
    response.headers['Content-type'] = 'text/csv'

  return response

<|endoftext|>


<commit_before># (c) 2017, XYSec Labs

from collections import namedtuple

def mapper_json_api(model: type, resource: dict) -> object:
    """
    Returns an obj of type `model` from dictified JSON `resource` for JSON APIs
    """
    attr = dict()
    for field in model._fields:
        if field == 'id':
            attr[field] = resource['data']['id']
        else:
            attr[field] = resource['data']['attributes'][
                field.replace('_', '-')]
    return model(**attr)

def mapper_drf_api(model: type, resource: dict) -> object:
    """
    Returns an obj of type `model` from dictified JSON `resource` for DRF APIs
    """
    accepted_params = {k: resource[k] for k in model._fields}
    return model(**accepted_params)

User = namedtuple(
    'User',
    ['id', 'email', 'first_name', 'lang', 'last_name', 'username']
)

Whoami = namedtuple(
    'Whoami',
    ['id', 'email', 'username', 'default_organization']
)

Organization = namedtuple(
    'Organization',
    ['id', 'name']
)

Project = namedtuple(
    'Project',
    ['id', 'created_on', 'file_count', 'package_name', 'platform',
     'updated_on']
)

File = namedtuple(
    'File',
    ['id', 'name', 'version', 'version_code', 'static_scan_progress']
)

Submission = namedtuple(
    'Submission',
    ['id', 'status', 'file', 'package_name', 'created_on', 'reason']
)

Analysis = namedtuple(
    'Analysis',
    ['id', 'risk', 'status', 'cvss_base', 'cvss_vector', 'cvss_version',
     'cvss_metrics_humanized', 'findings', 'updated_on', 'vulnerability',
     'owasp', 'pcidss', 'hipaa']
)

Vulnerability = namedtuple(
    'Vulnerability',
    ['id', 'name', 'description', 'intro', 'related_to',
     'business_implication', 'compliant', 'non_compliant', 'types']
)

OWASP = namedtuple(
    'OWASP',
    ['id', 'code', 'title', 'description', 'year']
)

PCIDSS = namedtuple(
    'PCIDSS',
    ['id', 'code', 'title', 'description']
)

PersonalToken = namedtuple(
    'AccessToken',
    ['name', 'key']
)
<commit_msg>Added compliances CWE, MSTG, ASVS<commit_after># (c) 2017, XYSec Labs

from collections import namedtuple

def mapper_json_api(model: type, resource: dict) -> object:
    """
    Returns an obj of type `model` from dictified JSON `resource` for JSON APIs
    """
    attr = dict()
    for field in model._fields:
        if field == 'id':
            attr[field] = resource['data']['id']
        else:
            attr[field] = resource['data']['attributes'][
                field.replace('_', '-')]
    return model(**attr)

def mapper_drf_api(model: type, resource: dict) -> object:
    """
    Returns an obj of type `model` from dictified JSON `resource` for DRF APIs
    """
    accepted_params = {k: resource[k] for k in model._fields}
    return model(**accepted_params)

User = namedtuple(
    'User',
    ['id', 'email', 'first_name', 'lang', 'last_name', 'username']
)

Whoami = namedtuple(
    'Whoami',
    ['id', 'email', 'username', 'default_organization']
)

Organization = namedtuple(
    'Organization',
    ['id', 'name']
)

Project = namedtuple(
    'Project',
    ['id', 'created_on', 'file_count', 'package_name', 'platform',
     'updated_on']
)

File = namedtuple(
    'File',
    ['id', 'name', 'version', 'version_code', 'static_scan_progress']
)

Submission = namedtuple(
    'Submission',
    ['id', 'status', 'file', 'package_name', 'created_on', 'reason']
)

Analysis = namedtuple(
    'Analysis',
    ['id', 'risk', 'status', 'cvss_base', 'cvss_vector', 'cvss_version',
     'cvss_metrics_humanized', 'findings', 'updated_on', 'vulnerability',
     'owasp', 'cwe', 'mstg', 'asvs']
)

Vulnerability = namedtuple(
    'Vulnerability',
    ['id', 'name', 'description', 'intro', 'related_to',
     'business_implication', 'compliant', 'non_compliant', 'types']
)

OWASP = namedtuple(
    'OWASP',
    ['id', 'code', 'title', 'description', 'year']
)

PCIDSS = namedtuple(
    'PCIDSS',
    ['id', 'code', 'title', 'description']
)

PersonalToken = namedtuple(
    'AccessToken',
    ['name', 'key']
)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# A collection of mimetypes to help with parsing and correctly handling/sending content
# based on their extensions.

__all__ = ['mime_types', 'file_exts']

_mimetypes = {
	'hqx':		['application/mac-binhex40'],
	'cpt':		['application/mac-compactpro'],
	'csv':		['text/x-comma-separated-values', 'text/comma-separated-values', 'application/octet-stream', 'application/vnd.ms-excel', 'text/csv', 'application/csv', 'application/excel', 'application/vnd.msexcel'],
	'bin':		['application/macbinary'],
	'dms':		['application/octet-stream'],
	'lha':		['application/octet-stream'],
	'lzh':		['application/octet-stream'],
	'exe':		['application/octet-stream'],
	'class':	['application/octet-stream'],
	'psd':		['application/x-photoshop'],
	'so':		['application/octet-stream'],
	'sea':		['application/octet-stream'],
	'dll':		['application/octet-stream'],
	'oda':		['application/oda'],
	'pdf':		['application/pdf', 'application/x-download'],
	'ai':		['application/postscript'],
	'eps':		['application/postscript'],
	'ps':		['application/postscript'],
	'smi':		['application/smil'],
	'smil':		['application/smil'],
	'mif':		['application/vnd.mif'],
	'xls':		['application/excel', 'application/vnd.ms-excel', 'application/msexcel'],
	'ppt':		['application/powerpoint', 'application/vnd.ms-powerpoint'],
	'wbxml':	['application/wbxml'],
	'wmlc':		['application/wmlc'],
	'dcr':		['application/x-director'],
	'dir':		['application/x-director'],
	'dxr':		['application/x-director'],
	'dvi':		['application/x-dvi'],
	'gtar':		['application/x-gtar'],
	'gz':		['application/x-gzip'],
	'php':		['application/x-httpd-php'],
	'php4':		['application/x-httpd-php'],
	'php3':		['application/x-httpd-php'],
	'phtml':	['application/x-httpd-php'],
	'phps':		['application/x-httpd-php-source'],
	'js':		['application/x-javascript'],
	'swf':		['application/x-shockwave-flash'],
	'sit':		['application/x-stuffit'],
	'tar':		['application/x-tar'],
	'tgz':		['application/x-tar'],
	'json':     ['application/json'],
	'xhtml':	['application/xhtml+xml'],
	'xht':		['application/xhtml+xml'],
	'zip':  	['application/x-zip', 'application/zip', 'application/x-zip-compressed'],
	'mid':		['audio/midi'],
	'midi':		['audio/midi'],
	'mpga':		['audio/mpeg'],
	'mp2':		['audio/mpeg'],
	'mp3':		['audio/mpeg', 'audio/mpg'],
	'aif':		['audio/x-aiff'],
	'aiff':		['audio/x-aiff'],
	'aifc':		['audio/x-aiff'],
	'ram':		['audio/x-pn-realaudio'],
	'rm':		['audio/x-pn-realaudio'],
	'rpm':		['audio/x-pn-realaudio-plugin'],
	'ra':		['audio/x-realaudio'],
	'rv':		['video/vnd.rn-realvideo'],
	'wav':		['audio/x-wav'],
	'bmp':		['image/bmp'],
	'gif':		['image/gif'],
	'jpeg':		['image/jpeg', 'image/pjpeg'],
	'jpg':		['image/jpeg', 'image/pjpeg'],
	'jpe':		['image/jpeg', 'image/pjpeg'],
	'png':		['image/png',  'image/x-png'],
	'tiff':		['image/tiff'],
	'tif':		['image/tiff'],
	'css':		['text/css'],
	'html':		['text/html'],
	'htm':		['text/html'],
	'shtml':	['text/html'],
	'txt':		['text/plain'],
	'text':		['text/plain'],
	'log':		['text/plain', 'text/x-log'],
	'rtx':		['text/richtext'],
	'rtf':		['text/rtf'],
	'xml':		['text/xml'],
	'xsl':		['text/xml'],
	'mpeg':		['video/mpeg'],
	'mpg':		['video/mpeg'],
	'mpe':		['video/mpeg'],
	'qt':		['video/quicktime'],
	'mov':		['video/quicktime'],
	'avi':		['video/x-msvideo'],
	'movie':	['video/x-sgi-movie'],
	'doc':		['application/msword'],
	'docx':		['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
	'xlsx':		['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
	'word':		['application/msword', 'application/octet-stream'],
	'xl':		['application/excel'],
	'eml':		['message/rfc822']
};

def mime_types(extension=None):
    """Returns the mimetypes for the given extension"""
    return _mimetypes.get(extension) if extension else None

def file_exts(mimetype=None):
    """Returns possible extensions for the given mimetype"""
    exts = []
    for ext in _mimetypes:
        if mimetype in _mimetypes[ext]:
            exts.append(ext)
    return exts
<commit_msg>small corrections<commit_after>#!/usr/bin/env python

# A collection of mimetypes to help with parsing and correctly handling/sending content
# based on their extensions.

__all__ = ['get_mimes', 'get_extensions']

_mimetypes = {
	'hqx':		['application/mac-binhex40'],
	'cpt':		['application/mac-compactpro'],
	'csv':		['text/x-comma-separated-values', 'text/comma-separated-values', 'application/octet-stream', 'application/vnd.ms-excel', 'text/csv', 'application/csv', 'application/excel', 'application/vnd.msexcel'],
	'bin':		['application/macbinary'],
	'dms':		['application/octet-stream'],
	'lha':		['application/octet-stream'],
	'lzh':		['application/octet-stream'],
	'exe':		['application/octet-stream'],
	'class':	['application/octet-stream'],
	'psd':		['application/x-photoshop'],
	'so':		['application/octet-stream'],
	'sea':		['application/octet-stream'],
	'dll':		['application/octet-stream'],
	'oda':		['application/oda'],
	'pdf':		['application/pdf', 'application/x-download'],
	'ai':		['application/postscript'],
	'eps':		['application/postscript'],
	'ps':		['application/postscript'],
	'smi':		['application/smil'],
	'smil':		['application/smil'],
	'mif':		['application/vnd.mif'],
	'xls':		['application/excel', 'application/vnd.ms-excel', 'application/msexcel'],
	'ppt':		['application/powerpoint', 'application/vnd.ms-powerpoint'],
	'wbxml':	['application/wbxml'],
	'wmlc':		['application/wmlc'],
	'dcr':		['application/x-director'],
	'dir':		['application/x-director'],
	'dxr':		['application/x-director'],
	'dvi':		['application/x-dvi'],
	'gtar':		['application/x-gtar'],
	'gz':		['application/x-gzip'],
	'php':		['application/x-httpd-php'],
	'php4':		['application/x-httpd-php'],
	'php3':		['application/x-httpd-php'],
	'phtml':	['application/x-httpd-php'],
	'phps':		['application/x-httpd-php-source'],
	'js':		['application/x-javascript'],
	'swf':		['application/x-shockwave-flash'],
	'sit':		['application/x-stuffit'],
	'tar':		['application/x-tar'],
	'tgz':		['application/x-tar'],
	'json':     ['application/json'],
	'xhtml':	['application/xhtml+xml'],
	'xht':		['application/xhtml+xml'],
	'zip':  	['application/x-zip', 'application/zip', 'application/x-zip-compressed'],
	'mid':		['audio/midi'],
	'midi':		['audio/midi'],
	'mpga':		['audio/mpeg'],
	'mp2':		['audio/mpeg'],
	'mp3':		['audio/mpeg', 'audio/mpg'],
	'aif':		['audio/x-aiff'],
	'aiff':		['audio/x-aiff'],
	'aifc':		['audio/x-aiff'],
	'ram':		['audio/x-pn-realaudio'],
	'rm':		['audio/x-pn-realaudio'],
	'rpm':		['audio/x-pn-realaudio-plugin'],
	'ra':		['audio/x-realaudio'],
	'rv':		['video/vnd.rn-realvideo'],
	'wav':		['audio/x-wav'],
	'bmp':		['image/bmp'],
	'gif':		['image/gif'],
	'jpeg':		['image/jpeg', 'image/pjpeg'],
	'jpg':		['image/jpeg', 'image/pjpeg'],
	'jpe':		['image/jpeg', 'image/pjpeg'],
	'png':		['image/png',  'image/x-png'],
	'tiff':		['image/tiff'],
	'tif':		['image/tiff'],
	'css':		['text/css'],
	'html':		['text/html'],
	'htm':		['text/html'],
	'shtml':	['text/html'],
	'txt':		['text/plain'],
	'text':		['text/plain'],
	'log':		['text/plain', 'text/x-log'],
	'rtx':		['text/richtext'],
	'rtf':		['text/rtf'],
	'xml':		['text/xml'],
	'xsl':		['text/xml'],
	'mpeg':		['video/mpeg'],
	'mpg':		['video/mpeg'],
	'mpe':		['video/mpeg'],
	'qt':		['video/quicktime'],
	'mov':		['video/quicktime'],
	'avi':		['video/x-msvideo'],
	'movie':	['video/x-sgi-movie'],
	'doc':		['application/msword'],
	'docx':		['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
	'xlsx':		['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
	'word':		['application/msword', 'application/octet-stream'],
	'xl':		['application/excel'],
	'eml':		['message/rfc822']
};

def get_mimes(extension=None):
    """Returns the mimetypes for the given extension"""
    return _mimetypes.get(extension) if extension else None

def get_extensions(mimetype=None):
    """Returns possible extensions for the given mimetype"""
    exts = []
    for ext in _mimetypes:
        if mimetype in _mimetypes[ext]:
            exts.append(ext)
    return exts
<|endoftext|>


<commit_before>#!/usr/bin/python
'''
Sript to ingest historical gdelt data and save to s3.
'''

# Import Dependencies

from __future__ import print_function
from __future__ import unicode_literals
import os
import io
import sys
import time
import zipfile
import requests
import argparse
import datetime
import pandas as pd
import requests
import lxml.html as lh
import numpy as np
import re
from tqdm import tqdm
import pickle
import fastparquet
import datetime
import dotenv

import boto3
import boto
# from boto.s3.key import Key
# from boto.s3.connection import S3Connection

def get_list_of_urls():
    '''
    Gets a list of gdelt archives in the form of URLs.
    INPUT:
        - NA
    OUTPUT:
        - masterlist_urls: List
                           URLs to obtain zipped CSVs.
    '''
    # Base directory for gdelt version 2 files
    gdelt_2_events_base_url = 'http://data.gdeltproject.org/gdeltv2/'

    # Directory containing all urls of historical data
    masterList_page = requests.get(gdelt_2_events_base_url+'masterfilelist.txt')
    print(type(masterList_page))

    # Requests output as text
    masterList = masterList_page.text
    print(len(masterList))

    # Extract URLs from text with regex
    masterList_urls = re.findall(r'(http|ftp|https)(://)([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?',
                                 masterList)
    print(len(masterList_urls))

    # Concatenate regex groups to unified url strings
    masterList_urls = list(map(lambda x: ''.join(x), masterList_urls))
    print(masterList_urls[0])

    return masterList_urls

def download_zip_in_chunks(directory, url):
    '''
    Downloads a zipped file in chunks.
    INPUT:
        - directory: String
                     Directory to write downloaded file.
        - url: String
               URL of the file to download.
    OUTPUT:
        - local_file: String
                      Filepath where contents were written.

    '''
    # Unique name of file at the end of a URL
    base_file = os.path.basename(url)
    print('\n\nDOWNOADING {}'.format(base_file))

    # Path to save output
    temp_path = directory

    if base_file in os.listdir(temp_path):
        print("FILE EXISTS, CONTINUING...")
    else:
        try:
            local_file = os.path.join(temp_path, base_file)
            # Connect to url
            req = requests.get(url, stream=True)

            # Open local file and write content to it
            with io.open(local_file, 'wb') as fp:
                # Stream data from requests generator in units of 1000
                for chunk in tqdm(req.iter_content(chunk_size=1000)):
                    # Write chunk content if found
                    if chunk:
                        fp.write(chunk)
        # Print errors and corresponding urls
        except Exception as e:
            print("ERROR: {}; {}".format(e, url))

def unzip_file(directory, zipped_file):
    '''
    Unzips files that have been downloaded locally and stores them in a directory.
    INPUT:
        - directory: String
                     Destination to save output.
        - zipped_file: String
                       Filepath to find zipped file for unzipping
    OUTPUT:
        - out_path: String
                    Filepath where content was written
    '''
    print('\n\nUnzipping {}'.format(zipped_file))
    # Try to unzip file
    try:
        # instatiate ZipFile object
        z = zipfile.ZipFile(zipped_file)
        # for each unit in z
        for name in tqdm(z.namelist()):
            # Open file and set output path
            f = z.open(name)
            out_path = os.path.join(directory, name)
            # Open out_path file for writing in utf-8
            with io.open(out_path, 'w', encoding='utf-8') as out_file:
                # Read unzipped content content as utf-8
                content = f.read().decode('utf-8')
                # Write content to output file
                out_file.write(content)

        print('Finished Unzippling {}'.format(zipped_file))
        return out_path

    # Print errors
    except zipfile.BadZipfile:
        print('Bad zip file for {}, passing on...'.format(zipped_file))

def make_gdelt_dataframe(csv_path):
    '''
    Takes a filepath, determines what type of gdelt data it is, and returns
    a pandas DataFrame.
    INPUT:
        - csv_path : string
    OUTPUT:
        - pandas dataframe
    '''

    if re.search(r'gkg', csv_path):
        columns = pickle.load(open(os.path.join(unzip_filepath,
                                                'gkg_columns.pkl'
                                                ),
                                   'rb')
                              )
    elif re.search(r'export', csv_path):
        columns = pickle.load(open(os.path.join(unzip_filepath,
                                                'events_columns.pkl'
                                                ),
                                   'rb')
                              )

    elif re.search(r'mentions', csv_path):
        columns = pickle.load(open(os.path.join(unzip_filepath,
                                                'mentions_columns.pkl'
                                                ),
                                   'rb')
                              )
    else:
        return 'NOT A VALID GDELT FILE'

    df = pd.read_csv(csv_path, sep='\t')
    df.columns = columns

    return df

def csv_to_parquet(file_path, output_path):
    '''
    Takes a csv path and makes a pandas DataFrame out of it so it can be
    compressed as a parqeut file.
    '''
    print('\n\nCONVERTING {} TO PARQUET'.format(file_path))

    df = make_gdelt_dataframe(file_path)
    basename = os.path.basename(file_path)
    output_filename = '{}.gzip.parquet'.format(basename)
    if basename in os.listdir(output_path):
        print("FILE EXISTS, CONTINUING...")
        return output_filename
    else:
        output_filepath = os.path.join(output_path, output_filename)
        fastparquet.write(output_filepath, df, compression='GZIP')

        return output_filepath

def parquet_to_s3(parquet_path):

    pf = fastparquet.ParquetFile(parquet_path)

    # Setup firehose connection
    firehose = boto3.client('firehose', region_name='us-west-2')

    firehose.put_record(DeliveryStreamName='gdelt-firehose',\
    Record={'Data': pf})

def upload_file_to_s3(csv_path):
    print('\n\nUPLOADING {} TO S3'.format(csv_path))

    s3 = boto.connect_s3()

    if re.search(r'gkg', csv_path):
        bucket_name = 'gdelt-streaming'
    elif re.search(r'export', csv_path):
        bucket_name = 'gdelt-streaming-events'
    elif re.search(r'mentions', csv_path):
        bucket_name = 'gdelt-streaming-mentions'
    else:
        bucket_name = 'gdelt-streaming'

    bucket = s3.lookup(bucket_name)

    current_keys = bucket.get_all_keys()
    current_keys = [k.key for k in current_keys]

    if os.path.basename(csv_path) in current_keys:
        print('FILE FOUND IN S3, CONTINUING...')
        return

    else:

        key_name = os.path.basename(csv_path)

        key = bucket.new_key(key_name)

        now = datetime.datetime.now()

        if re.search(r'gkg', csv_path):
            dataset = 'gdelt_gkg'
        elif re.search(r'export', csv_path):
            dataset = 'gdelt_events'
        elif re.search(r'mentions', csv_path):
            dataset = 'gdelt_mentions'
        else:
            dataset = 'other'

        metadata = {'dataset': dataset, 'upload_time': str(now)}

        key.metadata.update(metadata)

        key.set_contents_from_filename(csv_path)

        print('{} UPLOAD SUCCESSFUL'.format(key_name))

def get_most_recent_files_to_s3():
    gdelt_last_15 = requests.get('http://data.gdeltproject.org/gdeltv2/lastupdate.txt').text

    urls = re.findall(r'(?P<url>https?://[^\s]+)', gdelt_last_15)

    for f in tqdm(urls):

        f_zip_path = download_zip_in_chunks(download_filepath, f)

        f_unzip_path = unzip_file(unzip_filepath,
                                    os.path.join(download_filepath,
                                                 os.path.basename(f)))

        df = make_gdelt_dataframe(os.path.join(f_unzip_path))

        parquet = csv_to_parquet(f_unzip_path, parquet_filepath)

        upload_status = upload_file_to_s3(parquet)
        print(upload_status)

    delete_data_after_s3_upload()
def delete_data_after_s3_upload():
    raw_path = os.path.join(PROJ_ROOT, 'data/raw')
    interim_path = os.path.join(PROJ_ROOT, 'data/interim')
    parquet_path = os.path.join(PROJ_ROOT, 'data/parquet')

    file_paths = [raw_path, interim_path, parquet_path]

    print("REMOVING LEFTOVER FILES")
    for p in tqdm(file_paths):
        for f in os.listdir(p):
            os.remove(os.path.join(p, f))

if __name__ == '__main__':
    PROJ_ROOT = os.path.join(os.pardir, os.pardir)
    download_filepath = os.path.join(PROJ_ROOT, 'data/raw')
    unzip_filepath = os.path.join(PROJ_ROOT, 'data/interim')
    parquet_filepath = os.path.join(PROJ_ROOT, 'data/parquet')

    dotenv_path = os.path.join(PROJ_ROOT, '.env')
    dotenv.load_dotenv(dotenv_path)

    AWS_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY')
    AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
    host = os.environ.get('host')

    s3 = boto.connect_s3(host=host)

    # url_list = get_list_of_urls()
    # gkg_url = 'http://data.gdeltproject.org/gdeltv2/20150220183000.gkg.csv.zip'
    # gkg_zip_path = download_zip_in_chunks(download_filepath, gkg_url)
    # gkg_unzip_path = unzip_file(unzip_filepath,
    #                             os.path.join(download_filepath,
    #                                          os.path.basename(gkg_url)
    #                                          )
    #                             )
    #
    # mentions_url = 'http://data.gdeltproject.org/gdeltv2/20150220184500.mentions.CSV.zip'
    # mentions_zip_path = download_zip_in_chunks(download_filepath, mentions_url)
    # mentions_unzip_path = unzip_file(unzip_filepath,
    #                                  os.path.join(download_filepath,
    #                                               os.path.basename(mentions_url)
    #                                               )
    #                                  )
    #
    # events_url = 'http://data.gdeltproject.org/gdeltv2/20150220184500.export.CSV.zip'
    # events_zip_path = download_zip_in_chunks(download_filepath, events_url)
    # mentions_unzip_path = unzip_file(unzip_filepath,
    #                                  os.path.join(download_filepath,
    #                                               os.path.basename(events_url)
    #                                               )
    #                                 )
    #
    #
    gkg_columns = ['GKGRECORDID', 'DATE', 'SourceCollectionIdentifier',
              'SourceCommonName', 'DocumentIdentifier', 'Counts',
              'V2Counts', 'Themes', 'V2Themes', 'Locations',
              'V2Locations', 'Persons', 'V2Persons', 'Organizations',
              'V2Organizations', 'V2Tone', 'Dates', 'GCAM',
               'SharingImage', 'RelatedImages', 'SocialImageEmbeds',
              'SocialVideoEmbeds', 'Quotations', 'AllNames', 'Amounts',
              'TranslationInfo', 'Extras']

    events_columns = ['GLOBALEVENTID', 'SQLDATE', 'MonthYear', 'Year', 'FractionDate',
       'Actor1Code', 'Actor1Name', 'Actor1CountryCode',
       'Actor1KnownGroupCode', 'Actor1EthnicCode', 'Actor1Religion1Code',
       'Actor1Religion2Code', 'Actor1Type1Code', 'Actor1Type2Code',
       'Actor1Type3Code', 'Actor2Code', 'Actor2Name', 'Actor2CountryCode',
       'Actor2KnownGroupCode', 'Actor2EthnicCode', 'Actor2Religion1Code',
       'Actor2Religion2Code', 'Actor2Type1Code', 'Actor2Type2Code',
       'Actor2Type3Code', 'IsRootEvent', 'EventCode', 'EventBaseCode',
       'EventRootCode', 'QuadClass', 'GoldsteinScale', 'NumMentions',
       'NumSources', 'NumArticles', 'AvgTone', 'Actor1Geo_Type',
       'Actor1Geo_FullName', 'Actor1Geo_CountryCode', 'Actor1Geo_ADM1Code',
       'Actor1Geo_ADM2Code',
       'Actor1Geo_Lat', 'Actor1Geo_Long', 'Actor1Geo_FeatureID',
       'Actor2Geo_Type', 'Actor2Geo_FullName', 'Actor2Geo_CountryCode',
       'Actor2Geo_ADM1Code',
       'Actor2Geo_ADM2Code',
        'Actor2Geo_Lat', 'Actor2Geo_Long',
       'Actor2Geo_FeatureID', 'ActionGeo_Type', 'ActionGeo_FullName',
       'ActionGeo_CountryCode', 'ActionGeo_ADM1Code',
       'ActionGeo_ADM2Code',
        'ActionGeo_Lat',
       'ActionGeo_Long', 'ActionGeo_FeatureID', 'DATEADDED', 'SOURCEURL']

    mentions_columns = ['GLOBALEVENTID', 'EventTimeDate', 'MentionTimeDate',
                           'MentionType', 'MentionSourceName', 'MentionIdentifier',
                           'SentenceID', 'Actor1CharOffset', 'Actor2CharOffset',
                           'ActionCharOffset', 'InRawText', 'Confidence',
                           'MentionDocLen', 'MentionDocTone',
                           'MentionDocTranslationInfo', 'Extras']

    pickle.dump(gkg_columns, open(os.path.join(unzip_filepath,
                                               'gkg_columns.pkl'
                                               ),
                                  'wb'
                                  )
                )

    pickle.dump(events_columns, open(os.path.join(unzip_filepath,
                                                  'events_columns.pkl'
                                                  ),
                                     'wb'
                                     )
                )

    pickle.dump(mentions_columns, open(os.path.join(unzip_filepath,
                                                    'mentions_columns.pkl'
                                                    ),
                                       'wb'
                                       )
                )
    #
    # gkg = make_gdelt_dataframe(os.path.join(PROJ_ROOT, 'data/raw/20150220183000.gkg.csv'))
    # events = make_gdelt_dataframe(os.path.join(PROJ_ROOT, 'data/raw/20150220184500.export.CSV'))
    # mentions = make_gdelt_dataframe(os.path.join(PROJ_ROOT, 'data/raw/20150220184500.mentions.CSV'))

    # csv_to_parquet(os.path.join(PROJ_ROOT,'data/raw/20150220183000.gkg.csv'),
    #                             parquet_filepath)
    #
    # p = '../../data/parquet/20150220183000.gkg.csv.gzip.parquet'
    #
    # csv = '../../data/interim/20150220183000.gkg.csv'

    #upload_file_to_s3(p,'gdelt-streaming')

    get_most_recent_files_to_s3()
<commit_msg>get_gdelt.py only works if its run within its directory<commit_after>#!/usr/bin/python
'''
Sript to ingest historical gdelt data and save to s3.
'''
# pickle.dump(gkg_columns, open('~/gdelt_distributed_architecture/data/interim/gkg_columns.pkl', 'wb'))

# Import Dependencies

from __future__ import print_function
from __future__ import unicode_literals
import os
import io
import sys
import time
import zipfile
import requests
import argparse
import datetime
import pandas as pd
import requests
import lxml.html as lh
import numpy as np
import re
from tqdm import tqdm
import pickle
import fastparquet
import datetime
import dotenv

import boto3
import boto
# from boto.s3.key import Key
# from boto.s3.connection import S3Connection

def get_list_of_urls():
    '''
    Gets a list of gdelt archives in the form of URLs.
    INPUT:
        - NA
    OUTPUT:
        - masterlist_urls: List
                           URLs to obtain zipped CSVs.
    '''
    # Base directory for gdelt version 2 files
    gdelt_2_events_base_url = 'http://data.gdeltproject.org/gdeltv2/'

    # Directory containing all urls of historical data
    masterList_page = requests.get(gdelt_2_events_base_url+'masterfilelist.txt')
    print(type(masterList_page))

    # Requests output as text
    masterList = masterList_page.text
    print(len(masterList))

    # Extract URLs from text with regex
    masterList_urls = re.findall(r'(http|ftp|https)(://)([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?',
                                 masterList)
    print(len(masterList_urls))

    # Concatenate regex groups to unified url strings
    masterList_urls = list(map(lambda x: ''.join(x), masterList_urls))
    print(masterList_urls[0])

    return masterList_urls

def download_zip_in_chunks(directory, url):
    '''
    Downloads a zipped file in chunks.
    INPUT:
        - directory: String
                     Directory to write downloaded file.
        - url: String
               URL of the file to download.
    OUTPUT:
        - local_file: String
                      Filepath where contents were written.

    '''
    # Unique name of file at the end of a URL
    base_file = os.path.basename(url)
    print('\n\nDOWNOADING {}'.format(base_file))

    # Path to save output
    temp_path = directory

    if base_file in os.listdir(temp_path):
        print("FILE EXISTS, CONTINUING...")
    else:
        try:
            local_file = os.path.join(temp_path, base_file)
            # Connect to url
            req = requests.get(url, stream=True)

            # Open local file and write content to it
            with io.open(local_file, 'wb') as fp:
                # Stream data from requests generator in units of 1000
                for chunk in tqdm(req.iter_content(chunk_size=1000)):
                    # Write chunk content if found
                    if chunk:
                        fp.write(chunk)
        # Print errors and corresponding urls
        except Exception as e:
            print("ERROR: {}; {}".format(e, url))

def unzip_file(directory, zipped_file):
    '''
    Unzips files that have been downloaded locally and stores them in a directory.
    INPUT:
        - directory: String
                     Destination to save output.
        - zipped_file: String
                       Filepath to find zipped file for unzipping
    OUTPUT:
        - out_path: String
                    Filepath where content was written
    '''
    print('\n\nUnzipping {}'.format(zipped_file))
    # Try to unzip file
    try:
        # instatiate ZipFile object
        z = zipfile.ZipFile(zipped_file)
        # for each unit in z
        for name in tqdm(z.namelist()):
            # Open file and set output path
            f = z.open(name)
            out_path = os.path.join(directory, name)
            # Open out_path file for writing in utf-8
            with io.open(out_path, 'w', encoding='utf-8') as out_file:
                # Read unzipped content content as utf-8
                content = f.read().decode('utf-8')
                # Write content to output file
                out_file.write(content)

        print('Finished Unzippling {}'.format(zipped_file))
        return out_path

    # Print errors
    except zipfile.BadZipfile:
        print('Bad zip file for {}, passing on...'.format(zipped_file))

def make_gdelt_dataframe(csv_path):
    '''
    Takes a filepath, determines what type of gdelt data it is, and returns
    a pandas DataFrame.
    INPUT:
        - csv_path : string
    OUTPUT:
        - pandas dataframe
    '''

    if re.search(r'gkg', csv_path):
        columns = pickle.load(open(os.path.join(unzip_filepath,
                                                'gkg_columns.pkl'
                                                ),
                                   'rb')
                              )
    elif re.search(r'export', csv_path):
        columns = pickle.load(open(os.path.join(unzip_filepath,
                                                'events_columns.pkl'
                                                ),
                                   'rb')
                              )

    elif re.search(r'mentions', csv_path):
        columns = pickle.load(open(os.path.join(unzip_filepath,
                                                'mentions_columns.pkl'
                                                ),
                                   'rb')
                              )
    else:
        return 'NOT A VALID GDELT FILE'

    df = pd.read_csv(csv_path, sep='\t')
    df.columns = columns

    return df

def csv_to_parquet(file_path, output_path):
    '''
    Takes a csv path and makes a pandas DataFrame out of it so it can be
    compressed as a parqeut file.
    '''
    print('\n\nCONVERTING {} TO PARQUET'.format(file_path))

    df = make_gdelt_dataframe(file_path)
    basename = os.path.basename(file_path)
    output_filename = '{}.gzip.parquet'.format(basename)
    if basename in os.listdir(output_path):
        print("FILE EXISTS, CONTINUING...")
        return output_filename
    else:
        output_filepath = os.path.join(output_path, output_filename)
        fastparquet.write(output_filepath, df, compression='GZIP')

        return output_filepath

def parquet_to_s3(parquet_path):

    pf = fastparquet.ParquetFile(parquet_path)

    # Setup firehose connection
    firehose = boto3.client('firehose', region_name='us-west-2')

    firehose.put_record(DeliveryStreamName='gdelt-firehose',\
    Record={'Data': pf})

def upload_file_to_s3(csv_path):
    print('\n\nUPLOADING {} TO S3'.format(csv_path))

    s3 = boto.connect_s3()

    if re.search(r'gkg', csv_path):
        bucket_name = 'gdelt-streaming'
    elif re.search(r'export', csv_path):
        bucket_name = 'gdelt-streaming-events'
    elif re.search(r'mentions', csv_path):
        bucket_name = 'gdelt-streaming-mentions'
    else:
        bucket_name = 'gdelt-streaming'

    bucket = s3.lookup(bucket_name)

    current_keys = bucket.get_all_keys()
    current_keys = [k.key for k in current_keys]

    if os.path.basename(csv_path) in current_keys:
        print('FILE FOUND IN S3, CONTINUING...')
        return

    else:

        key_name = os.path.basename(csv_path)

        key = bucket.new_key(key_name)

        now = datetime.datetime.now()

        if re.search(r'gkg', csv_path):
            dataset = 'gdelt_gkg'
        elif re.search(r'export', csv_path):
            dataset = 'gdelt_events'
        elif re.search(r'mentions', csv_path):
            dataset = 'gdelt_mentions'
        else:
            dataset = 'other'

        metadata = {'dataset': dataset, 'upload_time': str(now)}

        key.metadata.update(metadata)

        key.set_contents_from_filename(csv_path)

        print('{} UPLOAD SUCCESSFUL'.format(key_name))

def get_most_recent_files_to_s3():
    gdelt_last_15 = requests.get('http://data.gdeltproject.org/gdeltv2/lastupdate.txt').text

    urls = re.findall(r'(?P<url>https?://[^\s]+)', gdelt_last_15)

    for f in tqdm(urls):

        f_zip_path = download_zip_in_chunks(download_filepath, f)

        f_unzip_path = unzip_file(unzip_filepath,
                                    os.path.join(download_filepath,
                                                 os.path.basename(f)))

        df = make_gdelt_dataframe(os.path.join(f_unzip_path))

        parquet = csv_to_parquet(f_unzip_path, parquet_filepath)

        upload_status = upload_file_to_s3(parquet)
        print(upload_status)

    delete_data_after_s3_upload()
def delete_data_after_s3_upload():
    raw_path = os.path.join(PROJ_ROOT, 'data/raw')
    interim_path = os.path.join(PROJ_ROOT, 'data/interim')
    parquet_path = os.path.join(PROJ_ROOT, 'data/parquet')

    file_paths = [raw_path, interim_path, parquet_path]

    print("REMOVING LEFTOVER FILES")
    for p in tqdm(file_paths):
        for f in os.listdir(p):
            os.remove(os.path.join(p, f))

if __name__ == '__main__':
    PROJ_ROOT = os.path.join(os.pardir, os.pardir)
    download_filepath = os.path.join(PROJ_ROOT, 'data/raw')
    unzip_filepath = os.path.join(PROJ_ROOT, 'data/interim')
    parquet_filepath = os.path.join(PROJ_ROOT, 'data/parquet')

    dotenv_path = os.path.join(PROJ_ROOT, '.env')
    dotenv.load_dotenv(dotenv_path)

    AWS_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY')
    AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
    host = os.environ.get('host')

    s3 = boto.connect_s3(host=host)

    # url_list = get_list_of_urls()
    # gkg_url = 'http://data.gdeltproject.org/gdeltv2/20150220183000.gkg.csv.zip'
    # gkg_zip_path = download_zip_in_chunks(download_filepath, gkg_url)
    # gkg_unzip_path = unzip_file(unzip_filepath,
    #                             os.path.join(download_filepath,
    #                                          os.path.basename(gkg_url)
    #                                          )
    #                             )
    #
    # mentions_url = 'http://data.gdeltproject.org/gdeltv2/20150220184500.mentions.CSV.zip'
    # mentions_zip_path = download_zip_in_chunks(download_filepath, mentions_url)
    # mentions_unzip_path = unzip_file(unzip_filepath,
    #                                  os.path.join(download_filepath,
    #                                               os.path.basename(mentions_url)
    #                                               )
    #                                  )
    #
    # events_url = 'http://data.gdeltproject.org/gdeltv2/20150220184500.export.CSV.zip'
    # events_zip_path = download_zip_in_chunks(download_filepath, events_url)
    # mentions_unzip_path = unzip_file(unzip_filepath,
    #                                  os.path.join(download_filepath,
    #                                               os.path.basename(events_url)
    #                                               )
    #                                 )
    #
    #
    gkg_columns = ['GKGRECORDID', 'DATE', 'SourceCollectionIdentifier',
              'SourceCommonName', 'DocumentIdentifier', 'Counts',
              'V2Counts', 'Themes', 'V2Themes', 'Locations',
              'V2Locations', 'Persons', 'V2Persons', 'Organizations',
              'V2Organizations', 'V2Tone', 'Dates', 'GCAM',
               'SharingImage', 'RelatedImages', 'SocialImageEmbeds',
              'SocialVideoEmbeds', 'Quotations', 'AllNames', 'Amounts',
              'TranslationInfo', 'Extras']

    events_columns = ['GLOBALEVENTID', 'SQLDATE', 'MonthYear', 'Year', 'FractionDate',
       'Actor1Code', 'Actor1Name', 'Actor1CountryCode',
       'Actor1KnownGroupCode', 'Actor1EthnicCode', 'Actor1Religion1Code',
       'Actor1Religion2Code', 'Actor1Type1Code', 'Actor1Type2Code',
       'Actor1Type3Code', 'Actor2Code', 'Actor2Name', 'Actor2CountryCode',
       'Actor2KnownGroupCode', 'Actor2EthnicCode', 'Actor2Religion1Code',
       'Actor2Religion2Code', 'Actor2Type1Code', 'Actor2Type2Code',
       'Actor2Type3Code', 'IsRootEvent', 'EventCode', 'EventBaseCode',
       'EventRootCode', 'QuadClass', 'GoldsteinScale', 'NumMentions',
       'NumSources', 'NumArticles', 'AvgTone', 'Actor1Geo_Type',
       'Actor1Geo_FullName', 'Actor1Geo_CountryCode', 'Actor1Geo_ADM1Code',
       'Actor1Geo_ADM2Code',
       'Actor1Geo_Lat', 'Actor1Geo_Long', 'Actor1Geo_FeatureID',
       'Actor2Geo_Type', 'Actor2Geo_FullName', 'Actor2Geo_CountryCode',
       'Actor2Geo_ADM1Code',
       'Actor2Geo_ADM2Code',
        'Actor2Geo_Lat', 'Actor2Geo_Long',
       'Actor2Geo_FeatureID', 'ActionGeo_Type', 'ActionGeo_FullName',
       'ActionGeo_CountryCode', 'ActionGeo_ADM1Code',
       'ActionGeo_ADM2Code',
        'ActionGeo_Lat',
       'ActionGeo_Long', 'ActionGeo_FeatureID', 'DATEADDED', 'SOURCEURL']

    mentions_columns = ['GLOBALEVENTID', 'EventTimeDate', 'MentionTimeDate',
                           'MentionType', 'MentionSourceName', 'MentionIdentifier',
                           'SentenceID', 'Actor1CharOffset', 'Actor2CharOffset',
                           'ActionCharOffset', 'InRawText', 'Confidence',
                           'MentionDocLen', 'MentionDocTone',
                           'MentionDocTranslationInfo', 'Extras']

    pickle.dump(gkg_columns, open(os.path.join(unzip_filepath,
                                               'gkg_columns.pkl'
                                               ),
                                  'wb'
                                  )
                )

    pickle.dump(events_columns, open(os.path.join(unzip_filepath,
                                                  'events_columns.pkl'
                                                  ),
                                     'wb'
                                     )
                )

    pickle.dump(mentions_columns, open(os.path.join(unzip_filepath,
                                                    'mentions_columns.pkl'
                                                    ),
                                       'wb'
                                       )
                )
    #
    # gkg = make_gdelt_dataframe(os.path.join(PROJ_ROOT, 'data/raw/20150220183000.gkg.csv'))
    # events = make_gdelt_dataframe(os.path.join(PROJ_ROOT, 'data/raw/20150220184500.export.CSV'))
    # mentions = make_gdelt_dataframe(os.path.join(PROJ_ROOT, 'data/raw/20150220184500.mentions.CSV'))

    # csv_to_parquet(os.path.join(PROJ_ROOT,'data/raw/20150220183000.gkg.csv'),
    #                             parquet_filepath)
    #
    # p = '../../data/parquet/20150220183000.gkg.csv.gzip.parquet'
    #
    # csv = '../../data/interim/20150220183000.gkg.csv'

    #upload_file_to_s3(p,'gdelt-streaming')

    get_most_recent_files_to_s3()
<|endoftext|>


<commit_before>from abc import ABCMeta, abstractmethod
import datetime
import cv2
import math
import os
import errno

class MediaWriter(object):
    """
    Abstract class for all media outputs. Forcing each inheritance
    to have a write class.
    """
    __metaclass__ = ABCMeta

    @abstractmethod
    def write(self, content, *args):
        """
        Write method to write media to disk
        :param media: the media to be written
        :param args: additional arguments that may be helpful
        """
        pass

class NullWriter(MediaWriter):
    def write(self, content, *args):
        pass

class ImageWriter(MediaWriter):
    """
    The ImageWriter will write an image to disk.
    """
    __metaclass__ = ABCMeta

    def __init__(self, prefix, file_format):
        """
        Default initializer
        :param prefix: the filename prefix a counter will be added
        after this string and incremented after each write to disk
        :param file_format: the file format for the images.
        """
        if not file_format.startswith('.'):
            file_format = '.' + file_format
        if prefix is not None:
            setup_dirs(prefix)
            self.name = prefix + file_format

    def write(self, img, *args):
        """
        Writes the given image to the location specified through the
        initializer
        :param img: the image that will be written to disk
        """
        cv2.imwrite(self.name % self.next_name(args), img)

    @abstractmethod
    def next_name(self, *args):
        """
        This abstract method returns the object that should be inserted
        into the filename
        :param args: the args, that is passed to write_image
        :return: the object that will be inserted into the filename
        """

class CustomImageWriter(ImageWriter):
    """
    Image Writer that uses a custom name. It takes it as the first
    argument in *args in the write method.
    """
    def __init__(self, prefix=None, file_format='.jpg'):
        """
        Default initializer
        :param prefix: the file location and file name prefix
        :param file_format: the file format e.g. .jpg, .png
        """
        super(CustomImageWriter, self).__init__(prefix + '%s', file_format)

    def next_name(self, *args):
        return args[0]

class IncrementalImageWriter(ImageWriter):
    """
    The IncrementalImageWriter will write an image to disk and append a
    number to the file name. This number will be auto-incremented by the
    specified step size after each write.
    """

    def __init__(self, prefix=None, file_format='.jpg', start=0, step=1):
        """
        Default initializer
        :param prefix: the file location and file name
        :param file_format: the file format e.g. .jpg, .png
        :param start: the starting number for the incremental count
        :param step: the step by which the count should increment
        """
        self.count = start - step
        self.step = step
        if prefix is not None:
            prefix += '%d'
        super(IncrementalImageWriter, self).__init__(prefix, file_format)

    def next_name(self, *args):
        self.count += self.step
        return self.count

class TimestampImageWriter(ImageWriter):
    """
    TimestampImageWriter is a ImageWriter that adds the timestamp of when
    the image was first shown in the original stream
    """

    def __init__(self, fps, prefix=None, file_format='.jpg'):
        """
        Default initializer
        :param fps: The number of frames per second in the original stream
        :param prefix: the prefix of the path to the output location
        :param file_format: the file format of the output image
        """
        self.fps = fps

        if prefix is not None:
            prefix += '%d'
        super(TimestampImageWriter, self).__init__(prefix, file_format)

    def next_name(self, args):
        current_frame = args[0]
        seconds = current_frame / self.fps
        milliseconds = seconds - math.floor(seconds)
        if milliseconds == 0:
            milliseconds = '000'
        else:
            milliseconds = str(int(milliseconds * (10 ** 3)))
        return str(datetime.timedelta(seconds=int(seconds))) + '.' + milliseconds.zfill(3)

class TimetableWriter(MediaWriter):
    """
    The Timetable Writer outputs each slide iteratively using
    the IncrementalImageWriter. Additionally it outputs a ".txt"
    document containing the slide name and their appearances.
    """
    def __init__(self, output_dir, timetable_loc, file_format):
        """
        Default initializer
        :param output_dir: the output directory for the sorted slides
        :param timetable_file: where the timetable file should be stored
        """
        setup_dirs(timetable_loc)
        self.timetable = open(timetable_loc, 'w')
        self.img_writer = IncrementalImageWriter(prefix=output_dir, start=1, file_format=file_format)
        self.txt_writer = TextWriter(timetable_loc)

    def write(self, slides, *args):
        i = 1
        for slide in slides:
            if slide.marked:
                self.txt_writer.write()
                continue
            self.img_writer.write(slide.img)
            appearances = slide.time
            for com in slide.times:
                appearances += " " + com
            self.txt_writer.write("Slide %d: %s\n" % (i, appearances))
            i += 1

        self.timetable.close()

class TextWriter(MediaWriter):
    def __init__(self, output_file):
        self.output_file = output_file

    def write(self, content, *args):
        self.output_file.write(content)

    def append(self, content, line):
        self.output_file.write()

def setup_dirs(path):
    """
    Takes a path and makes sure that directories to the path
    gets created and is writable.
    :param filename: the path to file
    """
    path = os.path.dirname(path)
    if path == '':
        return
    if not os.path.exists(path):
        try:
            os.makedirs(path)
        except OSError as exc:  # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise<commit_msg>Fixed a bug in TimestampWriter preventing it from writing to file<commit_after>from abc import ABCMeta, abstractmethod
import datetime
import cv2
import math
import os
import errno

class MediaWriter(object):
    """
    Abstract class for all media outputs. Forcing each inheritance
    to have a write class.
    """
    __metaclass__ = ABCMeta

    @abstractmethod
    def write(self, content, *args):
        """
        Write method to write media to disk
        :param media: the media to be written
        :param args: additional arguments that may be helpful
        """
        pass

class NullWriter(MediaWriter):
    def write(self, content, *args):
        pass

class ImageWriter(MediaWriter):
    """
    The ImageWriter will write an image to disk.
    """
    __metaclass__ = ABCMeta

    def __init__(self, prefix, file_format):
        """
        Default initializer
        :param prefix: the filename prefix a counter will be added
        after this string and incremented after each write to disk
        :param file_format: the file format for the images.
        """
        if not file_format.startswith('.'):
            file_format = '.' + file_format
        if prefix is not None:
            setup_dirs(prefix)
            self.name = prefix + file_format

    def write(self, img, *args):
        """
        Writes the given image to the location specified through the
        initializer
        :param img: the image that will be written to disk
        """
        cv2.imwrite(self.name % self.next_name(args), img)

    @abstractmethod
    def next_name(self, *args):
        """
        This abstract method returns the object that should be inserted
        into the filename
        :param args: the args, that is passed to write_image
        :return: the object that will be inserted into the filename
        """

class CustomImageWriter(ImageWriter):
    """
    Image Writer that uses a custom name. It takes it as the first
    argument in *args in the write method.
    """
    def __init__(self, prefix=None, file_format='.jpg'):
        """
        Default initializer
        :param prefix: the file location and file name prefix
        :param file_format: the file format e.g. .jpg, .png
        """
        super(CustomImageWriter, self).__init__(prefix + '%s', file_format)

    def next_name(self, *args):
        return args[0]

class IncrementalImageWriter(ImageWriter):
    """
    The IncrementalImageWriter will write an image to disk and append a
    number to the file name. This number will be auto-incremented by the
    specified step size after each write.
    """

    def __init__(self, prefix=None, file_format='.jpg', start=0, step=1):
        """
        Default initializer
        :param prefix: the file location and file name
        :param file_format: the file format e.g. .jpg, .png
        :param start: the starting number for the incremental count
        :param step: the step by which the count should increment
        """
        self.count = start - step
        self.step = step
        if prefix is not None:
            prefix += '%d'
        super(IncrementalImageWriter, self).__init__(prefix, file_format)

    def next_name(self, *args):
        self.count += self.step
        return self.count

class TimestampImageWriter(ImageWriter):
    """
    TimestampImageWriter is a ImageWriter that adds the timestamp of when
    the image was first shown in the original stream
    """

    def __init__(self, fps, prefix=None, file_format='.jpg'):
        """
        Default initializer
        :param fps: The number of frames per second in the original stream
        :param prefix: the prefix of the path to the output location
        :param file_format: the file format of the output image
        """
        self.fps = fps

        if prefix is not None:
            prefix += '%s'
        super(TimestampImageWriter, self).__init__(prefix, file_format)

    def next_name(self, args):
        current_frame = args[0]
        seconds = current_frame / self.fps
        milliseconds = seconds - math.floor(seconds)
        if milliseconds == 0:
            milliseconds = '000'
        else:
            milliseconds = str(int(milliseconds * (10 ** 3)))
        return str(datetime.timedelta(seconds=int(seconds))) + '.' + milliseconds.zfill(3)

class TimetableWriter(MediaWriter):
    """
    The Timetable Writer outputs each slide iteratively using
    the IncrementalImageWriter. Additionally it outputs a ".txt"
    document containing the slide name and their appearances.
    """
    def __init__(self, output_dir, timetable_loc, file_format):
        """
        Default initializer
        :param output_dir: the output directory for the sorted slides
        :param timetable_file: where the timetable file should be stored
        """
        setup_dirs(timetable_loc)
        self.timetable = open(timetable_loc, 'w')
        self.img_writer = IncrementalImageWriter(prefix=output_dir, start=1, file_format=file_format)
        self.txt_writer = TextWriter(timetable_loc)

    def write(self, slides, *args):
        i = 1
        for slide in slides:
            if slide.marked:
                self.txt_writer.write()
                continue
            self.img_writer.write(slide.img)
            appearances = slide.time
            for com in slide.times:
                appearances += " " + com
            self.txt_writer.write("Slide %d: %s\n" % (i, appearances))
            i += 1

        self.timetable.close()

class TextWriter(MediaWriter):
    def __init__(self, output_file):
        self.output_file = output_file

    def write(self, content, *args):
        self.output_file.write(content)

    def append(self, content, line):
        self.output_file.write()

def setup_dirs(path):
    """
    Takes a path and makes sure that directories to the path
    gets created and is writable.
    :param filename: the path to file
    """
    path = os.path.dirname(path)
    if path == '':
        return
    if not os.path.exists(path):
        try:
            os.makedirs(path)
        except OSError as exc:  # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
from collections import defaultdict
import numpy
import optimizer

class MegaTableau(object):

    """
    A representation of tableaux for manipulation by the maxent learner.
    Derived from a file of tab-delimited tableaux.
    Contains the following attributes:
        self.constraints -------- list of constraint names
            this is found on the first line of the input file
        self.constraints_abbrev - list of abbreviated constraint names
            this is found on the second line of the input file
            *DO NOT PUT CANDIDATES ON THE SECOND LINE OF THE INPUT FILE*
        self.weights ------------ a list of weights for constraints
        self.tableau ------------ a dictionary of dictionaries:
            {input: {output: [freq, violDic, maxentScore]}}
            freq = float()
            violDic = dictionary of constraint violations (integers).
                Keys are constraint indecies, based on order of constraints in self.constraints
            maxentScore = e**harmony. Initialized to zero (because harmony is undefined without weights).
    Contains the following methods:
        self.read_megt_file(megt_file) - moves the data from the .txt file to the attributes
            self.weights is not populated.
        self.read_weights_file(megt_file) - populates self.weights
    """

    def __init__(self, megt_file=None):
        """
        megt_file -- a tab-delimited file of tableaux
            follow the OTSoft guidelines for formatting.
            OTSoft guidelines are summarized in class descriptor too.
        """
        self.constraints = []
        #formerly self.constrainst_abbrev
        self.constraints_abbrev = []
        self.weights = []
        self.tableau = defaultdict(dict)
        if megt_file:
            self.read_megt_file(megt_file)

    def read_megt_file(self, megt_file):
        """
        Populates the following attributes with data from megt_file
            self.constraints -------- list of constraint names
            self.constraints_abbrev - list of abbreviations of constraint names
            self.tableau ------------ dictionary: {input: {output: [freq, violDic, maxentScore]}}
        megt_file: string representation of an OTSoft input file.
        """
        with open(megt_file) as f:
            #fstr = f.read().rstrip().split('\n') #making list of all rows
            self.constraints = f.readline().rstrip('\n').split('\t')[3:] #populating constraints
            self.constraints_abbrev = f.readline().rstrip('\n').split('\t')[3:]#populating constraint abbreviations
            self.weights = numpy.zeros(len(self.constraints)) # starting weights
            for line in f:
                splitline = line.rstrip('\n').split('\t')
                if len(splitline) > 3:
                    current_input = splitline[0] if splitline[0] else current_input
                    current_output = splitline[1]
                    if splitline[2]:
                        freq = float(splitline[2])
                    if not splitline[2]:
                        freq = 0.0
                    viol_vec = [int(v) if v else None for v in splitline[3:]]
                    violations = {}
                    for c in range(0,len(viol_vec)):   # use indecies for constraint keys
                        if viol_vec[c]:
                            violations[c] = viol_vec[c]
                    self.tableau[current_input][current_output] = [freq,violations,0] #frequency, violations, maxent_val

    def read_weights_file(self, weights_file):
        """
        Read in a file containing (potentially null) constraint weights.
        Each line in the file is either:
            (constraint name)\t(weight)
            or
            (weight)
        In the former case, weights can be in any order as long as the name-weight associations are right.
        In the latter case, the weights must be in the same order as in the MEGT input file.
        Files mixing the two conventions must be in the same order as in the MEGT input file.
        """
        with open(weights_file) as f:
            #to complain about not having all constraint names specified
            constraintFlags = []
            #to cope when constraint names are not specified
            counter = 0
            if len(self.weights) != len(self.constraints):
                print "please run self.read_megt_file() before self.read_weights_file()."
                return
            posDict = {constraint:pos for pos, constraint in enumerate(self.constraints)}
            for line in f:
                splitline = line.rstrip().split('\t')
                flag = False
                if len(splitline) == 1:
                    flag = True
                    if splitline[0]:
                        self.weights[counter] = float(splitline[0])
                if len(splitline) == 2:
                    if splitline[1]:
                        self.weights[posDict[splitline[0]]] = float(splitline[1])
                constraintFlags.append(flag)
                counter += 1
            if any(constraintFlags):
                for pos, conFlag in enumerate(constraintFlags):
                    if conFlag == True:
                        print "constraint", pos, "has no name in weight file, coping ..."

    def write_output(self, file_name):
        ''' Write a text file with the information in the megatableau object
        '''
        file = open(file_name,"w")

        # Create and sort list of input keys, and dictionary of lists of output keys.
        inp_keys = self.tableau.keys()
        inp_keys.sort()
        outp_keys = {}
        for i in inp_keys:
            outp_keys[i] = self.tableau[i].keys()
            outp_keys[i].sort()

        # Add 1st line with constraint names
        file.write("\t\t\t\t\t")
        for constraint in self.constraints:
            file.write(constraint+"\t")
        file.write("\n")

        # Add 2nd line with constraint abbreviations
        file.write("\t\t\t\t\t")
        for constraint_abbrev in self.constraints_abbrev:
            file.write(constraint_abbrev+"\t")
        file.write("\n")

        # Add 3rd line with constraint weights, headers
        file.write("\t\tObs\tExp\tprob\t")
        for weight in self.weights:
            file.write(str(round(weight,3))+"\t")
        file.write("\n")

        # Add inputs, outputs, violations
        for inp in inp_keys:
            file.write(inp) # Add input
            zscore = optimizer.z_score(self.tableau,inp)
            assert zscore != 0, "Can't print a tableau before updating its maxent values."
            total = 0
            for outp in outp_keys[inp]: # Count total occurances of this UR
                total += self.tableau[inp][outp][0]
            for outp in outp_keys[inp]:
                obs  = self.tableau[inp][outp][0]
                prob = self.tableau[inp][outp][2] / zscore
                exp  = prob * total     # Calculate expected counts
                file.write("\t"+outp+"\t")              # Add output
                file.write(str(obs)+"\t")               # Add observed counts
                file.write(str(round(exp, 1))+"\t")     # Add expected counts
                file.write(str(round(prob, 4))+"\t")    # Add probability
                for c in range(0,len(self.constraints)):# Add violations
                    if c in self.tableau[inp][outp][1].keys():
                        file.write(str(self.tableau[inp][outp][1][c]))
                    file.write("\t")
                file.write("\n")

        # Add data probability
        file.write("\nLog probability: ")
        file.write(str(-optimizer.neg_log_probability(self.weights, self.tableau)))

        # Done
        file.close()
<commit_msg>Fix megatableau.py breaking on breaker1<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
from collections import defaultdict
import numpy
import optimizer

class MegaTableau(object):

    """
    A representation of tableaux for manipulation by the maxent learner.
    Derived from a file of tab-delimited tableaux.
    Contains the following attributes:
        self.constraints -------- list of constraint names
            this is found on the first line of the input file
        self.constraints_abbrev - list of abbreviated constraint names
            this is found on the second line of the input file
            *DO NOT PUT CANDIDATES ON THE SECOND LINE OF THE INPUT FILE*
        self.weights ------------ a list of weights for constraints
        self.tableau ------------ a dictionary of dictionaries:
            {input: {output: [freq, violDic, maxentScore]}}
            freq = float()
            violDic = dictionary of constraint violations (integers).
                Keys are constraint indecies, based on order of constraints in self.constraints
            maxentScore = e**harmony. Initialized to zero (because harmony is undefined without weights).
    Contains the following methods:
        self.read_megt_file(megt_file) - moves the data from the .txt file to the attributes
            self.weights is not populated.
        self.read_weights_file(megt_file) - populates self.weights
    """

    def __init__(self, megt_file=None):
        """
        megt_file -- a tab-delimited file of tableaux
            follow the OTSoft guidelines for formatting.
            OTSoft guidelines are summarized in class descriptor too.
        """
        self.constraints = []
        #formerly self.constrainst_abbrev
        self.constraints_abbrev = []
        self.weights = []
        self.tableau = defaultdict(dict)
        if megt_file:
            self.read_megt_file(megt_file)

    def read_megt_file(self, megt_file):
        """
        Populates the following attributes with data from megt_file
            self.constraints -------- list of constraint names
            self.constraints_abbrev - list of abbreviations of constraint names
            self.tableau ------------ dictionary: {input: {output: [freq, violDic, maxentScore]}}
        megt_file: string representation of an OTSoft input file.
        """
        with open(megt_file) as f:
            #fstr = f.read().rstrip().split('\n') #making list of all rows
            self.constraints = f.readline().rstrip('\n').split('\t')[3:] #populating constraints
            self.constraints_abbrev = f.readline().rstrip('\n').split('\t')[3:]#populating constraint abbreviations
            self.weights = numpy.zeros(len(self.constraints)) # starting weights
            for line in f:
                splitline = line.rstrip('\n').split('\t')
                current_input = splitline[0] if splitline[0] else current_input
                current_output = splitline[1]
                if splitline[2]:
                    freq = float(splitline[2])
                if not splitline[2]:
                    freq = 0.0
                viol_vec = [int(v) if v else None for v in splitline[3:]]
                violations = {}
                for c in range(0,len(viol_vec)):   # use indecies for constraint keys
                    if viol_vec[c]:
                        violations[c] = viol_vec[c]
                self.tableau[current_input][current_output] = [freq,violations,0] #frequency, violations, maxent_val

    def read_weights_file(self, weights_file):
        """
        Read in a file containing (potentially null) constraint weights.
        Each line in the file is either:
            (constraint name)\t(weight)
            or
            (weight)
        In the former case, weights can be in any order as long as the name-weight associations are right.
        In the latter case, the weights must be in the same order as in the MEGT input file.
        Files mixing the two conventions must be in the same order as in the MEGT input file.
        """
        with open(weights_file) as f:
            #to complain about not having all constraint names specified
            constraintFlags = []
            #to cope when constraint names are not specified
            counter = 0
            if len(self.weights) != len(self.constraints):
                print "please run self.read_megt_file() before self.read_weights_file()."
                return
            posDict = {constraint:pos for pos, constraint in enumerate(self.constraints)}
            for line in f:
                splitline = line.rstrip().split('\t')
                flag = False
                if len(splitline) == 1:
                    flag = True
                    if splitline[0]:
                        self.weights[counter] = float(splitline[0])
                if len(splitline) == 2:
                    if splitline[1]:
                        self.weights[posDict[splitline[0]]] = float(splitline[1])
                constraintFlags.append(flag)
                counter += 1
            if any(constraintFlags):
                for pos, conFlag in enumerate(constraintFlags):
                    if conFlag == True:
                        print "constraint", pos, "has no name in weight file, coping ..."

    def write_output(self, file_name):
        ''' Write a text file with the information in the megatableau object
        '''
        file = open(file_name,"w")

        # Create and sort list of input keys, and dictionary of lists of output keys.
        inp_keys = self.tableau.keys()
        inp_keys.sort()
        outp_keys = {}
        for i in inp_keys:
            outp_keys[i] = self.tableau[i].keys()
            outp_keys[i].sort()

        # Add 1st line with constraint names
        file.write("\t\t\t\t\t")
        for constraint in self.constraints:
            file.write(constraint+"\t")
        file.write("\n")

        # Add 2nd line with constraint abbreviations
        file.write("\t\t\t\t\t")
        for constraint_abbrev in self.constraints_abbrev:
            file.write(constraint_abbrev+"\t")
        file.write("\n")

        # Add 3rd line with constraint weights, headers
        file.write("\t\tObs\tExp\tprob\t")
        for weight in self.weights:
            file.write(str(round(weight,3))+"\t")
        file.write("\n")

        # Add inputs, outputs, violations
        for inp in inp_keys:
            file.write(inp) # Add input
            zscore = optimizer.z_score(self.tableau,inp)
            assert zscore != 0, "Can't print a tableau before updating its maxent values."
            total = 0
            for outp in outp_keys[inp]: # Count total occurances of this UR
                total += self.tableau[inp][outp][0]
            for outp in outp_keys[inp]:
                obs  = self.tableau[inp][outp][0]
                prob = self.tableau[inp][outp][2] / zscore
                exp  = prob * total     # Calculate expected counts
                file.write("\t"+outp+"\t")              # Add output
                file.write(str(obs)+"\t")               # Add observed counts
                file.write(str(round(exp, 1))+"\t")     # Add expected counts
                file.write(str(round(prob, 4))+"\t")    # Add probability
                for c in range(0,len(self.constraints)):# Add violations
                    if c in self.tableau[inp][outp][1].keys():
                        file.write(str(self.tableau[inp][outp][1][c]))
                    file.write("\t")
                file.write("\n")

        # Add data probability
        file.write("\nLog probability: ")
        file.write(str(-optimizer.neg_log_probability(self.weights, self.tableau)))

        # Done
        file.close()
<|endoftext|>