<commit_before>import os,os.path,sys,datetime,subprocess,string
from time import strftime

#------------------------
#  2015-02-05  D. O'Hara
#   requires wget
#------------------------

#-----------------------------------
#- set configuration parms
#-----------------------------------

# The only part of this script that required hand-editing was the INPUT/OUTPUT
# directories.  Let us just read them from the command line.  This also
# allows people to enter a Windows-based path.
if len(sys.argv) < 3:
    print("usage: get_synpuf_files.py path/to/input/directory path/to/output/directory")
    print(sys.argv)
    quit();

INPUT_DIRECTORY     = sys.argv[1]
OUTPUT_DIRECTORY    = sys.argv[2]
if not os.path.exists(OUTPUT_DIRECTORY): os.mkdir(OUTPUT_DIRECTORY)

SAMPLE_RANGE = [4,15]

#-----------------------------------
#-----------------------------------
def get_timestamp():
    return strftime("%Y-%m-%d %H:%M:%S")

#-----------------------------------
#- download and unzip all the files for a sample
#  combine the 3 beneficiary files into 1 file
#-----------------------------------
def download_synpuf_files(sample_directory, sample_number):
    print('-'*80)
    print(get_timestamp(),' download_synpuf_files starting: sample_number=',sample_number)

    # as of 2015-02-06, files come from different places
    url_www_cms_gov        = 'www.cms.gov/Research-Statistics-Data-and-Systems/Downloadable-Public-Use-Files/SynPUFs/Downloads'
    url_downloads_cms_gov  = 'downloads.cms.gov/files'

    synpuf_files = [
        [ url_www_cms_gov,          'DE1_0_2008_Beneficiary_Summary_File_Sample_~~.zip'         ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Carrier_Claims_Sample_~~A.zip'          ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Carrier_Claims_Sample_~~B.zip'          ],
        [ url_www_cms_gov,          'DE1_0_2008_to_2010_Inpatient_Claims_Sample_~~.zip'         ],
        [ url_www_cms_gov,          'DE1_0_2008_to_2010_Outpatient_Claims_Sample_~~.zip'        ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Prescription_Drug_Events_Sample_~~.zip' ],
        [ url_www_cms_gov,          'DE1_0_2009_Beneficiary_Summary_File_Sample_~~.zip'         ],
        [ url_www_cms_gov,          'DE1_0_2010_Beneficiary_Summary_File_Sample_~~.zip'         ]
    ]

    download_directory = os.path.join(sample_directory,"DE_{0}".format(sample_number))
    if not os.path.exists(download_directory): os.mkdir(download_directory)

    for base_url,sp_file in synpuf_files:
        sp_file = sp_file.replace('~~',str(sample_number))
        file_url = 'http://{0}/{1}'.format(base_url, sp_file)
        file_local = os.path.join(download_directory,sp_file)
        print('..downloading -> ', file_url)

        rc = subprocess.call(['wget','-4',
                            file_url,
                            '-O', file_local])

        rc = subprocess.call(['unzip',
                              '-u',                             # update / create files
                              file_local,
                              '-d',download_directory])

    #-- combine the beneficiary files
    combine_beneficiary_files(download_directory, download_directory, sample_number)

    print(get_timestamp(),' Done')

#-----------------------------------
#- combine 3 beneficiary files into 1, with the year prefixed
#-----------------------------------
def combine_beneficiary_files(sample_directory, output_directory, sample_number):
    print('-'*80)
    print(get_timestamp(),' combine_beneficiary_files starting: sample_number=',sample_number)

    output_bene_filename = os.path.join(output_directory ,
                        'DE1_0_comb_Beneficiary_Summary_File_Sample_{0}.csv'.format(sample_number))

    print('Writing to ->',output_bene_filename)
    total_recs_in=0
    total_recs_out=0

    with open(output_bene_filename, 'w') as f_out:
        for year in ['2008','2009','2010']:
            input_bene_filename = os.path.join(sample_directory,
                            'DE1_0_{0}_Beneficiary_Summary_File_Sample_{1}.csv'.format(year,sample_number))
            print('Reading    ->',input_bene_filename)
            recs_in=0
            with open(input_bene_filename, 'r') as f_in:
                for line in f_in:
                    recs_in+=1
                    if recs_in % 25000 == 0: print('Year-{0}: records read ={1}, total written={2}'.format(year,recs_in, total_recs_out))
                    f_out.write(year + ',' + line)
                    total_recs_out+=1
            print('Year-{0}: total records read ={1}'.format(year,recs_in))
            total_recs_in+=recs_in

    print(get_timestamp(),' Done: total records read ={0}, total records written={1}'.format(total_recs_in, total_recs_out))

#-----------------------------------
#-----------------------------------
if __name__ == '__main__':

    print(get_timestamp(),' Combine Beneficiary Year files...starting')

    print('INPUT_DIRECTORY          =', INPUT_DIRECTORY)
    print('OUTPUT_DIRECTORY         =', OUTPUT_DIRECTORY)
    print('SAMPLE_RANGE             =', SAMPLE_RANGE)

    #------
    # download from CMS
    #------
    for sample_number in SAMPLE_RANGE:
        download_synpuf_files(INPUT_DIRECTORY, sample_number)

    print(get_timestamp(),' Done')
<commit_msg>Use os.makedirs to emulate 'mkdir -p' in POSIX<commit_after>import os,os.path,sys,datetime,subprocess,string
from time import strftime

#------------------------
#  2015-02-05  D. O'Hara
#   requires wget
#------------------------

#-----------------------------------
#- set configuration parms
#-----------------------------------

# The only part of this script that required hand-editing was the INPUT/OUTPUT
# directories.  Let us just read them from the command line.  This also
# allows people to enter a Windows-based path.
if len(sys.argv) < 3:
    print("usage: get_synpuf_files.py path/to/input/directory path/to/output/directory")
    print(sys.argv)
    quit();

INPUT_DIRECTORY     = sys.argv[1]
OUTPUT_DIRECTORY    = sys.argv[2]
if not os.path.exists(OUTPUT_DIRECTORY): os.makedirs(OUTPUT_DIRECTORY)

SAMPLE_RANGE = [4,15]

#-----------------------------------
#-----------------------------------
def get_timestamp():
    return strftime("%Y-%m-%d %H:%M:%S")

#-----------------------------------
#- download and unzip all the files for a sample
#  combine the 3 beneficiary files into 1 file
#-----------------------------------
def download_synpuf_files(sample_directory, sample_number):
    print('-'*80)
    print(get_timestamp(),' download_synpuf_files starting: sample_number=',sample_number)

    # as of 2015-02-06, files come from different places
    url_www_cms_gov        = 'www.cms.gov/Research-Statistics-Data-and-Systems/Downloadable-Public-Use-Files/SynPUFs/Downloads'
    url_downloads_cms_gov  = 'downloads.cms.gov/files'

    synpuf_files = [
        [ url_www_cms_gov,          'DE1_0_2008_Beneficiary_Summary_File_Sample_~~.zip'         ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Carrier_Claims_Sample_~~A.zip'          ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Carrier_Claims_Sample_~~B.zip'          ],
        [ url_www_cms_gov,          'DE1_0_2008_to_2010_Inpatient_Claims_Sample_~~.zip'         ],
        [ url_www_cms_gov,          'DE1_0_2008_to_2010_Outpatient_Claims_Sample_~~.zip'        ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Prescription_Drug_Events_Sample_~~.zip' ],
        [ url_www_cms_gov,          'DE1_0_2009_Beneficiary_Summary_File_Sample_~~.zip'         ],
        [ url_www_cms_gov,          'DE1_0_2010_Beneficiary_Summary_File_Sample_~~.zip'         ]
    ]

    download_directory = os.path.join(sample_directory,"DE_{0}".format(sample_number))
    if not os.path.exists(download_directory): os.makedirs(download_directory)

    for base_url,sp_file in synpuf_files:
        sp_file = sp_file.replace('~~',str(sample_number))
        file_url = 'http://{0}/{1}'.format(base_url, sp_file)
        file_local = os.path.join(download_directory,sp_file)
        print('..downloading -> ', file_url)

        rc = subprocess.call(['wget','-4',
                            file_url,
                            '-O', file_local])

        rc = subprocess.call(['unzip',
                              '-u',                             # update / create files
                              file_local,
                              '-d',download_directory])

    #-- combine the beneficiary files
    combine_beneficiary_files(download_directory, download_directory, sample_number)

    print(get_timestamp(),' Done')

#-----------------------------------
#- combine 3 beneficiary files into 1, with the year prefixed
#-----------------------------------
def combine_beneficiary_files(sample_directory, output_directory, sample_number):
    print('-'*80)
    print(get_timestamp(),' combine_beneficiary_files starting: sample_number=',sample_number)

    output_bene_filename = os.path.join(output_directory ,
                        'DE1_0_comb_Beneficiary_Summary_File_Sample_{0}.csv'.format(sample_number))

    print('Writing to ->',output_bene_filename)
    total_recs_in=0
    total_recs_out=0

    with open(output_bene_filename, 'w') as f_out:
        for year in ['2008','2009','2010']:
            input_bene_filename = os.path.join(sample_directory,
                            'DE1_0_{0}_Beneficiary_Summary_File_Sample_{1}.csv'.format(year,sample_number))
            print('Reading    ->',input_bene_filename)
            recs_in=0
            with open(input_bene_filename, 'r') as f_in:
                for line in f_in:
                    recs_in+=1
                    if recs_in % 25000 == 0: print('Year-{0}: records read ={1}, total written={2}'.format(year,recs_in, total_recs_out))
                    f_out.write(year + ',' + line)
                    total_recs_out+=1
            print('Year-{0}: total records read ={1}'.format(year,recs_in))
            total_recs_in+=recs_in

    print(get_timestamp(),' Done: total records read ={0}, total records written={1}'.format(total_recs_in, total_recs_out))

#-----------------------------------
#-----------------------------------
if __name__ == '__main__':

    print(get_timestamp(),' Combine Beneficiary Year files...starting')

    print('INPUT_DIRECTORY          =', INPUT_DIRECTORY)
    print('OUTPUT_DIRECTORY         =', OUTPUT_DIRECTORY)
    print('SAMPLE_RANGE             =', SAMPLE_RANGE)

    #------
    # download from CMS
    #------
    for sample_number in SAMPLE_RANGE:
        download_synpuf_files(INPUT_DIRECTORY, sample_number)

    print(get_timestamp(),' Done')
<|endoftext|>


<commit_before>"""See what metadata IDPGIS has to offer.

Run daily from RUN_2AM.sh
"""

import requests
from pyiem.util import get_dbconn, logger

LOG = logger()
SERVICE = (
    "https://idpgis.ncep.noaa.gov/arcgis/rest/services/NWS_Observations/"
    "ahps_riv_gauges/MapServer/0/query?where=1%3D1&"
    "geometryType=esriGeometryPoint&outFields=PEDTS%2CGaugeLID"
    "&returnGeometry=true&f=json"
)

def get_current(dbconn):
    """Get what we have."""
    current = {}
    cursor = dbconn.cursor()
    cursor.execute(
        "SELECT id, s.iemid, value from "
        "stations s LEFT JOIN station_attributes a "
        "on (s.iemid = a.iemid and a.attr = 'PEDTS') "
        "WHERE s.network ~* 'DCP'"
    )
    for row in cursor:
        current[row[0]] = {"pedts": row[2], "iemid": row[1]}
    cursor.close()
    return current

def get_idp():
    """See what AHPS has."""
    LOG.debug("Fetching %s", SERVICE)
    req = requests.get(SERVICE, timeout=60)
    jobj = req.json()
    idp = {}
    for feat in jobj["features"]:
        attrs = feat["attributes"]
        if attrs["pedts"] is None or attrs["pedts"] == "N/A":
            continue
        idp[attrs["gaugelid"]] = attrs["pedts"]
    return idp

def main():
    """Go Main Go."""
    idp = get_idp()
    with get_dbconn("mesosite") as dbconn:
        current = get_current(dbconn)
        for nwsli, pedts in idp.items():
            if nwsli not in current:
                LOG.info("station %s is unknown to IEM", nwsli)
                continue
            currentval = current[nwsli]["pedts"]
            if currentval == pedts:
                continue
            cursor = dbconn.cursor()
            if currentval is None:
                cursor.execute(
                    "INSERT into station_attributes(iemid, attr, value) "
                    "VALUES (%s, %s, %s)",
                    (current[nwsli]["iemid"], "PEDTS", pedts),
                )
            else:
                cursor.execute(
                    "UPDATE station_attributes SET value = %s WHERE "
                    "iemid = %s and attr = 'PEDTS'",
                    (pedts, current[nwsli]["iemid"]),
                )
            LOG.info("%s PEDTS %s -> %s", nwsli, currentval, pedts)
            cursor.close()
            dbconn.commit()

if __name__ == "__main__":
    main()
<commit_msg>add unknown stations found from IDP sync #278<commit_after>"""See what metadata IDPGIS has to offer.

Run daily from RUN_2AM.sh
"""

import requests
from pyiem.reference import nwsli2state
from pyiem.util import get_dbconn, logger

LOG = logger()
SERVICE = (
    "https://idpgis.ncep.noaa.gov/arcgis/rest/services/NWS_Observations/"
    "ahps_riv_gauges/MapServer/0/query?where=1%3D1&"
    "geometryType=esriGeometryPoint&"
    "outFields=PEDTS%2CGaugeLID%2CLocation%2CState&returnGeometry=true&f=json"
)

def get_current(dbconn):
    """Get what we have."""
    current = {}
    cursor = dbconn.cursor()
    cursor.execute(
        "SELECT id, s.iemid, value from "
        "stations s LEFT JOIN station_attributes a "
        "on (s.iemid = a.iemid and a.attr = 'PEDTS') "
        "WHERE s.network ~* 'DCP'"
    )
    for row in cursor:
        current[row[0]] = {"pedts": row[2], "iemid": row[1]}
    cursor.close()
    return current

def get_idp():
    """See what AHPS has."""
    LOG.debug("Fetching %s", SERVICE)
    req = requests.get(SERVICE, timeout=60)
    jobj = req.json()
    idp = {}
    for feat in jobj["features"]:
        attrs = feat["attributes"]
        attrs["lon"] = feat["geometry"]["x"]
        attrs["lat"] = feat["geometry"]["y"]
        idp[attrs["gaugelid"]] = attrs
    return idp

def add_station(dbconn, nwsli, attrs):
    """Add the station."""
    if len(nwsli) != 5 or nwsli2state.get(nwsli[-2:]) != attrs["state"]:
        LOG.info("Skipping %s as un-rectified metadata", nwsli)
        return None
    cursor = dbconn.cursor()
    cursor.execute(
        "INSERT into stations(id, name, plot_name, state, country, geom, "
        "network) VALUES (%s, %s, %s, %s, 'US', 'SRID=4326;POINT(%s %s)', "
        "%s) RETURNING iemid",
        (
            nwsli,
            attrs["location"],
            attrs["location"],
            attrs["state"],
            attrs["lon"],
            attrs["lat"],
            f"{attrs['state']}_DCP",
        ),
    )
    iemid = cursor.fetchone()[0]
    cursor.close()
    dbconn.commit()
    return iemid

def main():
    """Go Main Go."""
    idp = get_idp()
    with get_dbconn("mesosite") as dbconn:
        current = get_current(dbconn)
        for nwsli, attrs in idp.items():
            if nwsli not in current:
                iemid = add_station(dbconn, nwsli, attrs)
                if iemid is None:
                    continue
                LOG.info("Adding station %s[%s]", nwsli, iemid)
                current[nwsli] = {"pedts": "", "iemid": iemid}
            currentval = current[nwsli]["pedts"]
            if attrs["pedts"] is None or attrs["pedts"] == "N/A":
                continue
            if currentval == attrs["pedts"]:
                continue
            cursor = dbconn.cursor()
            if currentval is None:
                cursor.execute(
                    "INSERT into station_attributes(iemid, attr, value) "
                    "VALUES (%s, %s, %s)",
                    (current[nwsli]["iemid"], "PEDTS", attrs["pedts"]),
                )
            else:
                cursor.execute(
                    "UPDATE station_attributes SET value = %s WHERE "
                    "iemid = %s and attr = 'PEDTS'",
                    (attrs["pedts"], current[nwsli]["iemid"]),
                )
            LOG.info("%s PEDTS %s -> %s", nwsli, currentval, attrs["pedts"])
            cursor.close()
            dbconn.commit()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (C) 2018 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#

import sys
import os
import re
import subprocess
import jinja2
import socket
import time
import syslog as sl

from vyos.config import Config
from vyos import ConfigError

pidfile = r'/var/run/accel_pptp.pid'
pptp_cnf_dir = r'/etc/accel-ppp/pptp'
chap_secrets = pptp_cnf_dir + '/chap-secrets'
pptp_conf = pptp_cnf_dir + '/pptp.config'
# accel-pppd -d -c /etc/accel-ppp/pppoe/pppoe.config -p /var/run/accel_pppoe.pid

### config path creation
if not os.path.exists(pptp_cnf_dir):
  os.makedirs(pptp_cnf_dir)
  sl.syslog(sl.LOG_NOTICE, pptp_cnf_dir  + " created")

pptp_config = '''
### generated by accel_pptp.py ###
[modules]
log_syslog
pptp
ippool
chap-secrets
{% if authentication['auth_proto'] %}
{{ authentication['auth_proto'] }}
{% else %}
auth_mschap_v2
{% endif %}
{% if authentication['mode'] == 'radius' %}
radius
{% endif -%}

[core]
thread-count={{thread_cnt}}

[log]
syslog=accel-pptp,daemon
copy=1
level=5

{% if dns %}
[dns]
{% if dns[0] %}
dns1={{dns[0]}}
{% endif %}
{% if dns[1] %}
dns2={{dns[1]}}
{% endif %}
{% endif %}

{% if wins %}
[wins]
{% if wins[0] %}
wins1={{wins[0]}}
{% endif %}
{% if wins[1] %}
wins2={{wins[1]}}
{% endif %}
{% endif %}

[pptp]
bind={{outside_addr}}
verbose=5
ppp-max-mtu={{mtu}}

[client-ip-range]
0.0.0.0/0

[ip-pool]
tunnel={{client_ip_pool}}
gw-ip-address={{gw_ip}}

{% if authentication['mode'] == 'local' %}
[chap-secrets]
chap-secrets=/etc/accel-ppp/pptp/chap-secrets
{% endif %}

[ppp]
verbose=5
check-ip=1
single-session=replace

{% if authentication['mode'] == 'radius' %}
[radius]
{% for rsrv in authentication['radiussrv']: %}
server={{rsrv}},{{authentication['radiussrv'][rsrv]['secret']}},\
req-limit={{authentication['radiussrv'][rsrv]['req-limit']}},\
fail-time={{authentication['radiussrv'][rsrv]['fail-time']}}
{% endfor %}
timeout=30
acct-timeout=30
max-try=3
{%endif %}

[cli]
tcp=127.0.0.1:2003
'''

### pptp chap secrets
chap_secrets_conf = '''
# username  server  password  acceptable local IP addresses
{% for user in authentication['local-users'] %}
{% if authentication['local-users'][user]['state'] == 'enabled' %}
{{user}}\t*\t{{authentication['local-users'][user]['passwd']}}\t{{authentication['local-users'][user]['ip']}}
{% endif %}
{% endfor %}
'''
###
# inline helper functions
###
# depending on hw and threads, daemon needs a little to start
# if it takes longer than 100 * 0.5 secs, exception is being raised
# not sure if that's the best way to check it, but it worked so far quite well
###
def chk_con():
  cnt = 0
  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  while True:
    try:
      s.connect(("127.0.0.1", 2003))
      break
    except ConnectionRefusedError:
      time.sleep(0.5)
      cnt +=1
      if cnt == 100:
        raise("failed to start pptp server")
        break

### chap_secrets file if auth mode local
def write_chap_secrets(c):
  tmpl = jinja2.Template(chap_secrets_conf, trim_blocks=True)
  chap_secrets_txt = tmpl.render(c)
  old_umask = os.umask(0o077)
  open(chap_secrets,'w').write(chap_secrets_txt)
  os.umask(old_umask)
  sl.syslog(sl.LOG_NOTICE, chap_secrets + ' written')

def accel_cmd(cmd=''):
  if not cmd:
    return None
  try:
    ret = subprocess.check_output(['/usr/bin/accel-cmd','-p','2003',cmd]).decode().strip()
    return ret
  except:
    return 1

###
# inline helper functions end
###

def get_config():
  c = Config()
  if not c.exists('vpn pptp remote-access '):
    return None

  c.set_level('vpn pptp remote-access')
  config_data = {
    'authentication'  : {
        'mode'          : 'local',
        'local-users'   : {
        },
    'radiussrv'         : {},
    'auth_proto'        : 'auth_mschap_v2'
    },
    'outside_addr'    : '',
    'dns'             : [],
    'wins'            : [],
    'client_ip_pool'  : '',
    'mtu'   : '1436',
  }

  ### general options ###

  if c.exists('dns-servers server-1'):
    config_data['dns'].append( c.return_value('dns-servers server-1'))
  if c.exists('dns-servers server-2'):
    config_data['dns'].append( c.return_value('dns-servers server-2'))
  if c.exists('wins-servers server-1'):
    config_data['wins'].append( c.return_value('wins-servers server-1'))
  if c.exists('wins-servers server-2'):
    config_data['wins'].append( c.return_value('wins-servers server-2'))
  if c.exists('outside-address'):
    config_data['outside_addr'] = c.return_value('outside-address')

  ### auth local
  if c.exists('authentication mode local'):
    if c.exists('authentication local-users username'):
      for usr in c.list_nodes('authentication local-users username'):
        config_data['authentication']['local-users'].update(
          {
            usr : {
              'passwd' : '',
              'state'  : 'enabled',
              'ip'     : ''
            }
          }
        )

        if c.exists('authentication local-users username ' + usr + ' password'):
          config_data['authentication']['local-users'][usr]['passwd'] = c.return_value('authentication local-users username ' + usr + ' password')
        if c.exists('authentication local-users username ' + usr + ' disable'):
          config_data['authentication']['local-users'][usr]['state'] = 'disable'
        if c.exists('authentication local-users username ' + usr + ' static-ip'):
          config_data['authentication']['local-users'][usr]['ip'] = c.return_value('authentication local-users username ' + usr + ' static-ip')

  ### authentication mode radius servers and settings

  if c.exists('authentication mode radius'):
    config_data['authentication']['mode'] = 'radius'
    rsrvs = c.list_nodes('authentication radius server')
    for rsrv in rsrvs:
      if c.return_value('authentication radius server ' + rsrv + ' fail-time') == None:
        ftime = '0'
      else:
        ftime = str(c.return_value('authentication radius server ' + rsrv + ' fail-time'))
      if c.return_value('authentication radius-server ' + rsrv + ' req-limit') == None:
        reql = '0'
      else:
        reql = str(c.return_value('authentication radius server ' + rsrv + ' req-limit'))

      config_data['authentication']['radiussrv'].update(
        {
          rsrv  : {
            'secret'  : c.return_value('authentication radius server ' + rsrv + ' key'),
            'fail-time' : ftime,
            'req-limit' : reql
            }
        }
      )

  if c.exists('client-ip-pool'):
    if c.exists('client-ip-pool start'):
      config_data['client_ip_pool'] = c.return_value('client-ip-pool start')
    if c.exists('client-ip-pool stop'):
      config_data['client_ip_pool'] += '-' + re.search('[0-9]+$', c.return_value('client-ip-pool stop')).group(0)
  if c.exists('mtu'):
    config_data['mtu'] = c.return_value('mtu')

  ### gateway address
  if c.exists('gateway-address'):
    config_data['gw_ip'] = c.return_value('gateway-address')
  else:
    config_data['gw_ip'] = re.sub('[0-9]+$','1',config_data['client_ip_pool'])

  if c.exists('authentication require'):
    if c.return_value('authentication require') == 'pap':
      config_data['authentication']['auth_proto'] = 'auth_pap'
    if c.return_value('authentication require') == 'chap':
      config_data['authentication']['auth_proto'] = 'auth_chap_md5'
    if c.return_value('authentication require') == 'mschap':
      config_data['authentication']['auth_proto'] = 'auth_mschap_v1'
    if c.return_value('authentication require') == 'mschap-v2':
      config_data['authentication']['auth_proto'] = 'auth_mschap_v2'

  return config_data

def verify(c):
  if c == None:
    return None

  if c['authentication']['mode'] == 'local':
    if not c['authentication']['local-users']:
      raise ConfigError('pppoe-server authentication local-users required')
    for usr in c['authentication']['local-users']:
      if not c['authentication']['local-users'][usr]['passwd']:
        raise ConfigError('user ' + usr + ' requires a password')

  if c['authentication']['mode'] == 'radius':
    if len(c['authentication']['radiussrv']) == 0:
      raise ConfigError('radius server required')
    for rsrv in c['authentication']['radiussrv']:
      if c['authentication']['radiussrv'][rsrv]['secret'] == None:
        raise ConfigError('radius server ' + rsrv + ' needs a secret configured')

def generate(c):
  if c == None:
    return None

  ### accel-cmd reload doesn't work so any change results in a restart of the daemon
  try:
    if os.cpu_count() == 1:
      c['thread_cnt'] = 1
    else:
      c['thread_cnt'] = int(os.cpu_count()/2)
  except KeyError:
    if os.cpu_count() == 1:
      c['thread_cnt'] = 1
    else:
      c['thread_cnt'] = int(os.cpu_count()/2)

  tmpl = jinja2.Template(pptp_config, trim_blocks=True)
  config_text = tmpl.render(c)
  open(pptp_conf,'w').write(config_text)

  if c['authentication']['local-users']:
    write_chap_secrets(c)

  return c

def apply(c):
  if c == None:
    if os.path.exists(pidfile):
      accel_cmd('shutdown hard')
      if os.path.exists(pidfile):
        os.remove(pidfile)
    return None

  if not os.path.exists(pidfile):
    ret = subprocess.call(['/usr/sbin/accel-pppd','-c',pptp_conf,'-p',pidfile,'-d'])
    chk_con()
    if ret !=0 and os.path.exists(pidfile):
      os.remove(pidfile)
      raise ConfigError('accel-pppd failed to start')
  else:
    ### if gw ip changes, only restart doesn't work
    accel_cmd('restart')
    sl.syslog(sl.LOG_NOTICE, "reloading config via daemon restart")

if __name__ == '__main__':
  try:
    c = get_config()
    verify(c)
    generate(c)
    apply(c)
  except ConfigError as e:
    print(e)
    sys.exit(1)
<commit_msg>[pptp] implementing MPPE 128-bit stateless compression option<commit_after>#!/usr/bin/env python3
#
# Copyright (C) 2018 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#

import sys
import os
import re
import subprocess
import jinja2
import socket
import time
import syslog as sl

from vyos.config import Config
from vyos import ConfigError

pidfile = r'/var/run/accel_pptp.pid'
pptp_cnf_dir = r'/etc/accel-ppp/pptp'
chap_secrets = pptp_cnf_dir + '/chap-secrets'
pptp_conf = pptp_cnf_dir + '/pptp.config'
# accel-pppd -d -c /etc/accel-ppp/pppoe/pppoe.config -p /var/run/accel_pppoe.pid

### config path creation
if not os.path.exists(pptp_cnf_dir):
  os.makedirs(pptp_cnf_dir)
  sl.syslog(sl.LOG_NOTICE, pptp_cnf_dir  + " created")

pptp_config = '''
### generated by accel_pptp.py ###
[modules]
log_syslog
pptp
ippool
chap-secrets
{% if authentication['auth_proto'] %}
{{ authentication['auth_proto'] }}
{% else %}
auth_mschap_v2
{% endif %}
{% if authentication['mode'] == 'radius' %}
radius
{% endif -%}

[core]
thread-count={{thread_cnt}}

[log]
syslog=accel-pptp,daemon
copy=1
level=5

{% if dns %}
[dns]
{% if dns[0] %}
dns1={{dns[0]}}
{% endif %}
{% if dns[1] %}
dns2={{dns[1]}}
{% endif %}
{% endif %}

{% if wins %}
[wins]
{% if wins[0] %}
wins1={{wins[0]}}
{% endif %}
{% if wins[1] %}
wins2={{wins[1]}}
{% endif %}
{% endif %}

[pptp]
bind={{outside_addr}}
verbose=5
ppp-max-mtu={{mtu}}
mppe={{authentication['mppe']}}
echo-interval=10
echo-failure=3

[client-ip-range]
0.0.0.0/0

[ip-pool]
tunnel={{client_ip_pool}}
gw-ip-address={{gw_ip}}

{% if authentication['mode'] == 'local' %}
[chap-secrets]
chap-secrets=/etc/accel-ppp/pptp/chap-secrets
{% endif %}

[ppp]
verbose=5
check-ip=1
single-session=replace

{% if authentication['mode'] == 'radius' %}
[radius]
{% for rsrv in authentication['radiussrv']: %}
server={{rsrv}},{{authentication['radiussrv'][rsrv]['secret']}},\
req-limit={{authentication['radiussrv'][rsrv]['req-limit']}},\
fail-time={{authentication['radiussrv'][rsrv]['fail-time']}}
{% endfor %}
timeout=30
acct-timeout=30
max-try=3
{%endif %}

[cli]
tcp=127.0.0.1:2003
'''

### pptp chap secrets
chap_secrets_conf = '''
# username  server  password  acceptable local IP addresses
{% for user in authentication['local-users'] %}
{% if authentication['local-users'][user]['state'] == 'enabled' %}
{{user}}\t*\t{{authentication['local-users'][user]['passwd']}}\t{{authentication['local-users'][user]['ip']}}
{% endif %}
{% endfor %}
'''
###
# inline helper functions
###
# depending on hw and threads, daemon needs a little to start
# if it takes longer than 100 * 0.5 secs, exception is being raised
# not sure if that's the best way to check it, but it worked so far quite well
###
def chk_con():
  cnt = 0
  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  while True:
    try:
      s.connect(("127.0.0.1", 2003))
      break
    except ConnectionRefusedError:
      time.sleep(0.5)
      cnt +=1
      if cnt == 100:
        raise("failed to start pptp server")
        break

### chap_secrets file if auth mode local
def write_chap_secrets(c):
  tmpl = jinja2.Template(chap_secrets_conf, trim_blocks=True)
  chap_secrets_txt = tmpl.render(c)
  old_umask = os.umask(0o077)
  open(chap_secrets,'w').write(chap_secrets_txt)
  os.umask(old_umask)
  sl.syslog(sl.LOG_NOTICE, chap_secrets + ' written')

def accel_cmd(cmd=''):
  if not cmd:
    return None
  try:
    ret = subprocess.check_output(['/usr/bin/accel-cmd','-p','2003',cmd]).decode().strip()
    return ret
  except:
    return 1

###
# inline helper functions end
###

def get_config():
  c = Config()
  if not c.exists('vpn pptp remote-access '):
    return None

  c.set_level('vpn pptp remote-access')
  config_data = {
    'authentication'  : {
        'mode'          : 'local',
        'local-users'   : {
        },
    'radiussrv'         : {},
    'auth_proto'        : 'auth_mschap_v2',
    'mppe'              : 'require'
    },
    'outside_addr'    : '',
    'dns'             : [],
    'wins'            : [],
    'client_ip_pool'  : '',
    'mtu'   : '1436',
  }

  ### general options ###

  if c.exists('dns-servers server-1'):
    config_data['dns'].append( c.return_value('dns-servers server-1'))
  if c.exists('dns-servers server-2'):
    config_data['dns'].append( c.return_value('dns-servers server-2'))
  if c.exists('wins-servers server-1'):
    config_data['wins'].append( c.return_value('wins-servers server-1'))
  if c.exists('wins-servers server-2'):
    config_data['wins'].append( c.return_value('wins-servers server-2'))
  if c.exists('outside-address'):
    config_data['outside_addr'] = c.return_value('outside-address')

  ### auth local
  if c.exists('authentication mode local'):
    if c.exists('authentication local-users username'):
      for usr in c.list_nodes('authentication local-users username'):
        config_data['authentication']['local-users'].update(
          {
            usr : {
              'passwd' : '',
              'state'  : 'enabled',
              'ip'     : ''
            }
          }
        )

        if c.exists('authentication local-users username ' + usr + ' password'):
          config_data['authentication']['local-users'][usr]['passwd'] = c.return_value('authentication local-users username ' + usr + ' password')
        if c.exists('authentication local-users username ' + usr + ' disable'):
          config_data['authentication']['local-users'][usr]['state'] = 'disable'
        if c.exists('authentication local-users username ' + usr + ' static-ip'):
          config_data['authentication']['local-users'][usr]['ip'] = c.return_value('authentication local-users username ' + usr + ' static-ip')

  ### authentication mode radius servers and settings

  if c.exists('authentication mode radius'):
    config_data['authentication']['mode'] = 'radius'
    rsrvs = c.list_nodes('authentication radius server')
    for rsrv in rsrvs:
      if c.return_value('authentication radius server ' + rsrv + ' fail-time') == None:
        ftime = '0'
      else:
        ftime = str(c.return_value('authentication radius server ' + rsrv + ' fail-time'))
      if c.return_value('authentication radius-server ' + rsrv + ' req-limit') == None:
        reql = '0'
      else:
        reql = str(c.return_value('authentication radius server ' + rsrv + ' req-limit'))

      config_data['authentication']['radiussrv'].update(
        {
          rsrv  : {
            'secret'  : c.return_value('authentication radius server ' + rsrv + ' key'),
            'fail-time' : ftime,
            'req-limit' : reql
            }
        }
      )

  if c.exists('client-ip-pool'):
    if c.exists('client-ip-pool start'):
      config_data['client_ip_pool'] = c.return_value('client-ip-pool start')
    if c.exists('client-ip-pool stop'):
      config_data['client_ip_pool'] += '-' + re.search('[0-9]+$', c.return_value('client-ip-pool stop')).group(0)
  if c.exists('mtu'):
    config_data['mtu'] = c.return_value('mtu')

  ### gateway address
  if c.exists('gateway-address'):
    config_data['gw_ip'] = c.return_value('gateway-address')
  else:
    config_data['gw_ip'] = re.sub('[0-9]+$','1',config_data['client_ip_pool'])

  if c.exists('authentication require'):
    if c.return_value('authentication require') == 'pap':
      config_data['authentication']['auth_proto'] = 'auth_pap'
    if c.return_value('authentication require') == 'chap':
      config_data['authentication']['auth_proto'] = 'auth_chap_md5'
    if c.return_value('authentication require') == 'mschap':
      config_data['authentication']['auth_proto'] = 'auth_mschap_v1'
    if c.return_value('authentication require') == 'mschap-v2':
      config_data['authentication']['auth_proto'] = 'auth_mschap_v2'

    if c.exists('authentication mppe'):
      config_data['authentication']['mppe'] = c.return_value('authentication mppe')

  return config_data

def verify(c):
  if c == None:
    return None

  if c['authentication']['mode'] == 'local':
    if not c['authentication']['local-users']:
      raise ConfigError('pppoe-server authentication local-users required')
    for usr in c['authentication']['local-users']:
      if not c['authentication']['local-users'][usr]['passwd']:
        raise ConfigError('user ' + usr + ' requires a password')

  if c['authentication']['mode'] == 'radius':
    if len(c['authentication']['radiussrv']) == 0:
      raise ConfigError('radius server required')
    for rsrv in c['authentication']['radiussrv']:
      if c['authentication']['radiussrv'][rsrv]['secret'] == None:
        raise ConfigError('radius server ' + rsrv + ' needs a secret configured')

def generate(c):
  if c == None:
    return None

  ### accel-cmd reload doesn't work so any change results in a restart of the daemon
  try:
    if os.cpu_count() == 1:
      c['thread_cnt'] = 1
    else:
      c['thread_cnt'] = int(os.cpu_count()/2)
  except KeyError:
    if os.cpu_count() == 1:
      c['thread_cnt'] = 1
    else:
      c['thread_cnt'] = int(os.cpu_count()/2)

  tmpl = jinja2.Template(pptp_config, trim_blocks=True)
  config_text = tmpl.render(c)
  open(pptp_conf,'w').write(config_text)

  if c['authentication']['local-users']:
    write_chap_secrets(c)

  return c

def apply(c):
  if c == None:
    if os.path.exists(pidfile):
      accel_cmd('shutdown hard')
      if os.path.exists(pidfile):
        os.remove(pidfile)
    return None

  if not os.path.exists(pidfile):
    ret = subprocess.call(['/usr/sbin/accel-pppd','-c',pptp_conf,'-p',pidfile,'-d'])
    chk_con()
    if ret !=0 and os.path.exists(pidfile):
      os.remove(pidfile)
      raise ConfigError('accel-pppd failed to start')
  else:
    ### if gw ip changes, only restart doesn't work
    accel_cmd('restart')
    sl.syslog(sl.LOG_NOTICE, "reloading config via daemon restart")

if __name__ == '__main__':
  try:
    c = get_config()
    verify(c)
    generate(c)
    apply(c)
  except ConfigError as e:
    print(e)
    sys.exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

from shape_learning.shape_learner_manager import ShapeLearnerManager
from shape_learning.shape_learner import SettingsStruct
from shape_learning.shape_modeler import ShapeModeler #for normaliseShapeHeight()

import numpy
import matplotlib.pyplot as plt

from kivy.config import Config
Config.set('kivy', 'logger_enable', 0)
Config.write()

from kivy.app import App
from kivy.uix.widget import Widget
from kivy.graphics import Color, Ellipse, Line

from scipy import interpolate

import argparse
parser = argparse.ArgumentParser(description='Learn a collection of letters in parallel')
parser.add_argument('word', action="store",
                help='The word to be learnt')

numPoints_shapeModeler = 70

shapesLearnt = []
wordsLearnt = []
shapeLearners = []
currentWord = []
settings_shapeLearners = []
userInputCaptures = []

def downsampleShape(shape,numDesiredPoints,xyxyFormat=False):
    numPointsInShape = len(shape)/2
    if(xyxyFormat):
        #make xyxy format
        x_shape = shape[0::2]
        y_shape = shape[1::2]
    else:
        x_shape = shape[0:numPointsInShape]
        y_shape = shape[numPointsInShape:]

    if isinstance(x_shape,numpy.ndarray): #convert arrays to lists for interp1d
        x_shape = (x_shape.T).tolist()[0]
        y_shape = (y_shape.T).tolist()[0]

    #make shape have the same number of points as the shape_modeler
    t_current = numpy.linspace(0, 1, numPointsInShape)
    t_desired = numpy.linspace(0, 1, numDesiredPoints)
    f = interpolate.interp1d(t_current, x_shape, kind='cubic')
    x_shape = f(t_desired)
    f = interpolate.interp1d(t_current, y_shape, kind='cubic')
    y_shape = f(t_desired)

    shape = []
    shape[0:numPoints_shapeModeler] = x_shape
    shape[numPoints_shapeModeler:] = y_shape

    return shape

userShape = []
class MyPaintWidget(Widget):

    def on_touch_down(self, touch):
        with self.canvas:

            #self.canvas.clear()
            Color(1, 1, 0)
            d = 30.
            touch.ud['line'] = Line(points=(touch.x, touch.y))

    def on_touch_move(self, touch):
        global userShape
        touch.ud['line'].points += [touch.x, touch.y]
        userShape += [touch.x, -touch.y]

    def on_touch_up(self, touch):
        global userShape
        touch.ud['line'].points
        userShape = downsampleShape(userShape,numPoints_shapeModeler,xyxyFormat=True)

        shapeCentre = ShapeModeler.getShapeCentre(userShape)
        for i in range(len(wordToLearn)):
            if(shapeCentre[0] > (self.width/len(wordToLearn))*i):
                shapeIndex_demoFor = i

        shapeType = wordManager.shapeAtIndexInCurrentCollection(shapeIndex_demoFor)
        print('Received demo for letter ' + shapeType)

        userShape = numpy.reshape(userShape, (-1, 1)); #explicitly make it 2D array with only one column
        userShape = ShapeModeler.normaliseShapeHeight(numpy.array(userShape))

        shape = wordManager.respondToDemonstration(shapeIndex_demoFor, userShape)

        userShape = []
        self.canvas.remove(touch.ud['line'])
        showShape(shape, shapeIndex_demoFor)

class UserInputCapture(App):

    def build(self):
        self.painter = MyPaintWidget()
        return self.painter

    def on_start(self):
        with self.painter.canvas:
            print(self.painter.width)
            Color(1, 1, 0)
            d = 30.
            for i in range(len(wordToLearn)-1):
                x = (self.painter.width/len(wordToLearn))*(i+1)
                Line(points=(x, 0, x, self.painter.height))

###---------------------------------------------- WORD LEARNING SETTINGS

def generateSettings(shapeType):
    paramsToVary = [2];            #Natural number between 1 and numPrincipleComponents, representing which principle component to vary from the template
    initialBounds_stdDevMultiples = numpy.array([[-6, 6]]);  #Starting bounds for paramToVary, as multiples of the parameter's observed standard deviation in the dataset
    doGroupwiseComparison = True; #instead of pairwise comparison with most recent two shapes
    initialParamValue = numpy.NaN
    initialBounds = numpy.array([[numpy.NaN, numpy.NaN]])

    if shapeType == 'a':
        paramsToVary = [6]
        initialBounds_stdDevMultiples = numpy.array([[-3, 3]])
        datasetFile = datasetDirectory + '/a_noHook_dataset.txt'
        initialParamValue = 0.8;
    elif shapeType == 'c':
        paramToVary = 4
        initialBounds_stdDevMultiples = numpy.array([[-10, 10]])
        datasetFile = datasetDirectory + '/c_dataset.txt'

    elif shapeType == 'd':
        datasetFile = datasetDirectory + '/d_cursive_dataset.txt'
    elif shapeType == 'e':
        paramToVary = 3;
        initialBounds_stdDevMultiples = numpy.array([[-6, 14]])
        datasetFile = datasetDirectory + '/e_dataset.txt'
        #initialParamValue = 0.8
    elif shapeType == 'm':
        paramToVary = 6;
        initialBounds_stdDevMultiples = numpy.array([[-10, -6]])
        datasetFile = datasetDirectory + '/m_dataset.txt'
        initialParamValue = -0.5;#0.0
    elif shapeType == 'n':
        paramToVary = 7;
        datasetFile = datasetDirectory + '/n_dataset.txt'
        initialParamValue = 0.0
    elif shapeType == 'o':
        paramsToVary = [4]
        initialBounds_stdDevMultiples = numpy.array([[-3.5, 3]])
        datasetFile = datasetDirectory + '/o_dataset.txt'
    elif shapeType == 'r':
        paramToVary = 1
        datasetFile = datasetDirectory + '/r_print_dataset.txt'
    elif shapeType == 's':
        datasetFile = datasetDirectory + '/s_print_dataset.txt'
    elif shapeType == 'u':
        paramsToVary = [3]
        datasetFile = datasetDirectory + '/u_dataset.txt'
    elif shapeType == 'v':
        paramToVary = 6
        datasetFile = datasetDirectory + '/v_dataset.txt'
    elif shapeType == 'w':
        datasetFile = datasetDirectory + '/w_dataset.txt'
    else:
        raise RuntimeError("Dataset is not known for shape "+ shapeType)

    settings = SettingsStruct(shape_learning = shapeType,
            paramsToVary = paramsToVary, doGroupwiseComparison = True,
            datasetFile = datasetFile, initialBounds = initialBounds,
            initialBounds_stdDevMultiples = initialBounds_stdDevMultiples,
            initialParamValue = initialParamValue, minParamDiff = 0.4)
    return settings

def showShape(shape, shapeIndex):
    plt.figure(shapeIndex+1)
    plt.clf()
    ShapeModeler.normaliseAndShowShape(shape.path)

if __name__ == "__main__":
    #parse arguments
    args = parser.parse_args()
    wordToLearn = args.word

    import inspect
    fileName = inspect.getsourcefile(ShapeModeler)
    installDirectory = fileName.split('/lib')[0]
    datasetDirectory = installDirectory + '/share/shape_learning/letter_model_datasets/uji_pen_chars2'

    wordManager = ShapeLearnerManager(generateSettings)
    wordSeenBefore = wordManager.newCollection(wordToLearn)

    plt.ion()
    for i in range(len(wordToLearn)):
        shape = wordManager.startNextShapeLearner()
        showShape(shape, i)

    UserInputCapture().run()
<commit_msg>[learning_letters.py]appends userShape to data<commit_after>#!/usr/bin/env python
# coding: utf-8

from shape_learning.shape_learner_manager import ShapeLearnerManager
from shape_learning.shape_learner import SettingsStruct
from shape_learning.shape_modeler import ShapeModeler #for normaliseShapeHeight()

# Mapping of letter to their writing style (when several available in dataset)
styleMap = {
'a':'a2',
'b':'b1', # cursive style
'd':'d1',
'g':'g1',
'h':'h1',
'k':'k1',
'l':'l1',
'n':'n1',
'o':'o1',
'p':'p1',
'r':'r1',
's':'s1',
'u':'u1',
'v':'v1'
}

# Mapping between letters and adquate nb of principal components and initial eigen values
eigenValuesMap = {
'a':[ 0.8 , 6 ],
'b':[ 0.0 , 2 ],
'c':[ 0.0 , 4 ],
'd':[ 0.0 , 2 ],
'e':[ 0.0 , 3 ],
'f':[ 0.0 , 2 ],
'g':[ 0.0 , 2 ],
'h':[ 0.0 , 2 ],
'i':[ 0.0 , 2 ],
'j':[ 0.0 , 2 ],
'k':[ 0.0 , 2 ],
'l':[ 0.0 , 2 ],
'm':[-0.5 , 6 ],
'n':[ 0.0 , 7 ],
'o':[ 0.0 , 4 ],
'p':[ 0.0 , 2 ],
'q':[ 0.0 , 2 ],
'r':[ 0.0 , 1 ],
's':[ 0.0 , 2 ],
't':[ 0.0 , 2 ],
'u':[ 0.0 , 3 ],
'v':[ 0.0 , 6 ],
'w':[ 0.0 , 2 ],
'x':[ 0.0 , 2 ],
'y':[ 0.0 , 2 ],
'z':[ 0.0 , 2 ]
}

import numpy
import matplotlib.pyplot as plt

from kivy.config import Config
Config.set('kivy', 'logger_enable', 0)
Config.write()

from kivy.app import App
from kivy.uix.widget import Widget
from kivy.graphics import Color, Ellipse, Line

from scipy import interpolate

import argparse
parser = argparse.ArgumentParser(description='Learn a collection of letters in parallel')
parser.add_argument('word', action="store",
                help='The word to be learnt')

numPoints_shapeModeler = 70

shapesLearnt = []
wordsLearnt = []
shapeLearners = []
currentWord = []
settings_shapeLearners = []
userInputCaptures = []

def downsampleShape(shape,numDesiredPoints,xyxyFormat=False):
    numPointsInShape = len(shape)/2
    if(xyxyFormat):
        #make xyxy format
        x_shape = shape[0::2]
        y_shape = shape[1::2]
    else:
        x_shape = shape[0:numPointsInShape]
        y_shape = shape[numPointsInShape:]

    if isinstance(x_shape,numpy.ndarray): #convert arrays to lists for interp1d
        x_shape = (x_shape.T).tolist()[0]
        y_shape = (y_shape.T).tolist()[0]

    #make shape have the same number of points as the shape_modeler
    t_current = numpy.linspace(0, 1, numPointsInShape)
    t_desired = numpy.linspace(0, 1, numDesiredPoints)
    f = interpolate.interp1d(t_current, x_shape, kind='cubic')
    x_shape = f(t_desired)
    f = interpolate.interp1d(t_current, y_shape, kind='cubic')
    y_shape = f(t_desired)

    shape = []
    shape[0:numPoints_shapeModeler] = x_shape
    shape[numPoints_shapeModeler:] = y_shape

    return shape

userShape = []
class MyPaintWidget(Widget):

    def on_touch_down(self, touch):
        with self.canvas:

            #self.canvas.clear()
            Color(1, 1, 0)
            d = 30.
            touch.ud['line'] = Line(points=(touch.x, touch.y))

    def on_touch_move(self, touch):
        global userShape
        touch.ud['line'].points += [touch.x, touch.y]
        userShape += [touch.x, -touch.y]

    def on_touch_up(self, touch):
        global userShape
        touch.ud['line'].points
        userShape = downsampleShape(userShape,numPoints_shapeModeler,xyxyFormat=True)

        shapeCentre = ShapeModeler.getShapeCentre(userShape)

        for i in range(len(wordToLearn)):
            if(shapeCentre[0] > (self.width/len(wordToLearn))*i):
                shapeIndex_demoFor = i

        shapeType = wordManager.shapeAtIndexInCurrentCollection(shapeIndex_demoFor)

        print('Received demo for letter ' + shapeType)

        userShape = numpy.reshape(userShape, (-1, 1)); #explicitly make it 2D array with only one column
        userShape = ShapeModeler.normaliseShapeHeight(numpy.array(userShape))

        #------------This is to add the user's drawing to the data base:
        dataShape = ''
        for x in userShape:
            dataShape += ('%f'%x[0] + ' ')
        dataShape += '\n'

        if shapeType not in eigenValuesMap:
            raise RuntimeError("Dataset is not known for shape "+ shapeType)
        else:
            if shapeType in styleMap:
                shapeType = styleMap[shapeType]
            datasetFile = datasetDirectory + '/' + shapeType + '.dat'

            with open(datasetFile, 'rb') as f:
                l = f.readlines()[0]
                numExemples = (int)(l)
                l = '%i\n' % (numExemples+1)
                newData = l
            with open(datasetFile, 'rb') as f:
                ind = 0
                for l in f.readlines():
                    if ind>0:
                        newData = newData + l
                    ind+=1
            with open(datasetFile, 'wb') as f:
                f.write(newData)
                f.write(dataShape)
        #---------------------------------------------------------------

        shape = wordManager.respondToDemonstration(shapeIndex_demoFor, userShape)

        userShape = []
        self.canvas.remove(touch.ud['line'])
        showShape(shape, shapeIndex_demoFor)

class UserInputCapture(App):

    def build(self):
        self.painter = MyPaintWidget()
        return self.painter

    def on_start(self):
        with self.painter.canvas:
            print(self.painter.width)
            Color(1, 1, 0)
            d = 30.

            for i in range(len(wordToLearn)-1):
                x = (self.painter.width/len(wordToLearn))*(i+1)
                Line(points=(x, 0, x, self.painter.height))

###---------------------------------------------- WORD LEARNING SETTINGS

def generateSettings(shapeType):
    paramsToVary = [2];            #Natural number between 1 and numPrincipleComponents, representing which principle component to vary from the template
    initialBounds_stdDevMultiples = numpy.array([[-6, 6]]);  #Starting bounds for paramToVary, as multiples of the parameter's observed standard deviation in the dataset
    doGroupwiseComparison = True; #instead of pairwise comparison with most recent two shapes
    initialParamValue = numpy.NaN
    initialBounds = numpy.array([[numpy.NaN, numpy.NaN]])

    if shapeType not in eigenValuesMap:
        raise RuntimeError("Dataset is not known for shape "+ shapeType)
    else:
        initialParamValue = eigenValuesMap[shapeType][0]
        paramsToVary = [eigenValuesMap[shapeType][1]]
        if shapeType in styleMap:
            shapeType = styleMap[shapeType]
        datasetFile = datasetDirectory + '/' + shapeType + '.dat'

    settings = SettingsStruct(shape_learning = shapeType,
            paramsToVary = paramsToVary, doGroupwiseComparison = True,
            datasetFile = datasetFile, initialBounds = initialBounds,
            initialBounds_stdDevMultiples = initialBounds_stdDevMultiples,
            initialParamValue = initialParamValue, minParamDiff = 0.4)
    return settings

def showShape(shape,shapeIndex ):
    plt.figure(shapeIndex+1)
    plt.clf()
    ShapeModeler.normaliseAndShowShape(shape.path)

if __name__ == "__main__":
    #parse arguments
    args = parser.parse_args()
    wordToLearn = args.word

    import inspect
    fileName = inspect.getsourcefile(ShapeModeler)
    installDirectory = fileName.split('/lib')[0]
    datasetDirectory = installDirectory + '/share/shape_learning/letter_model_datasets/uji_pen_chars2'

    wordManager = ShapeLearnerManager(generateSettings)
    wordSeenBefore = wordManager.newCollection(wordToLearn)

    plt.ion()
    for i in range(len(wordToLearn)):
        shape = wordManager.startNextShapeLearner()
        showShape(shape, i)

    UserInputCapture().run()
<|endoftext|>


<commit_before>#!/usr/local/bin/python2.7

import csv
import glob
import sys
import os
import re
import bz2
from bokeh.io import hplot, vplot
from bokeh.plotting import figure, show, save
from bokeh.resources import CDN
from bokeh.embed import file_html

indir = "../data/"
outdir = "../"

header = [
	"Names",
	"Host Names",
#	"Publications",
	"Instruments/Surveys",
	"z",
	r"vHelio",
#	"$N_{\\rm h}$",
	"Claimed Type",
#	"Notes",
	"Plots",
	"Data"
	]

columnkey = [
	"name",
	"host",
#	"citations",
	"instruments",
	"redshift",
	"hvel",
#	"nh",
	"claimedtype",
#	"notes",
	"plot",
	"data"
	]

footer = [
	"Note: IAU name preferred",
	"* Uncertain",
#	"* discovery\n‡ sne type first proposed",
	"",
	"",
	"",
#	"Line of sight H column",
	"",
#	"",
	"",
	""
	]

if (len(columnkey) != len(header) or len(columnkey) != len(footer)):
	print 'Error: Header and/or footer not same length as key list.'
	sys.exit(0)

dataavaillink = "Y";

header = dict(zip(columnkey,header))
headerrow = [header[x] for x in columnkey]

bandcolors = [
	"indigo",
	"firebrick",
	"forestgreen",
	"red",
	"crimson",
	"indigo",
	"darkblue",
	"mediumvioletred",
	"pink",
	"#d82930",
	"orangered",
	"mediumvioletred",
	"mediumspringgreen",
	"orange",
	"chocolate",
	"darkorange",
]

bandcodes = [
	"u",
	"g",
	"r",
	"i",
	"z",
	"U",
	"B",
	"V",
	"R",
	"I",
	"G",
	"Y",
	"J",
	"H",
	"K"
]

bandnames = [
	"u",
	"g",
	"r",
	"i",
	"z",
	"U",
	"B",
	"V",
	"R",
	"I",
	"G",
	"Y",
	"J",
	"H",
	"K"
]

bandcolordict = dict(zip(bandcodes,bandcolors))
bandnamedict = dict(zip(bandcodes,bandnames))

def bandcolorf(color):
	if (color in bandcolordict):
		return bandcolordict[color]
	else:
		return 'black'

def bandnamef(code):
	if (code in bandnamedict):
		return bandnamedict[code]
	else:
		return code

catalogrows = []
for file in (sorted(glob.glob(indir + "*.bz2"), key=lambda s: s.lower()) + sorted(glob.glob(indir + "*.dat"), key=lambda s: s.lower())):
	print file
	filehead, ext = os.path.splitext(file)
	if ext == ".dat":
		tsvin = open(file,'rb')
	elif ext == ".bz2":
		tsvin = bz2.BZ2File(file,'rb')
	else:
		print "illegal file extension"
	tsvin = csv.reader(tsvin, delimiter='\t')

	catalog = dict(zip(columnkey,['' for _ in xrange(len(columnkey))]))

	table = []

	phototu = []
	phototime = []
	photoband = []
	photoinstrument = []
	photoAB = []
	photoerr = []
	phototype = []

	eventname = os.path.basename(file).split('.')[0]

	plotavail = False;
	for row in tsvin:
		if row[0] == 'photometry':
			plotavail = True;
			plotlink = "";
			catalog['plot'] = plotlink

			phototu.append(row[1])
			phototime.append(float(row[2]))
			photoband.append(row[4])
			photoinstrument.append(row[6].strip())
			photoAB.append(float(row[8]))
			if not row[10]:
				photoerr.append(0.)
			else:
				photoerr.append(float(row[10]))
			phototype.append(int(row[12]))

		if row[0] in columnkey:
			table.append(row)
			catalog[row[0]] = row[1]

	catalog['data'] = r'Download'

	instrulist = sorted(filter(None, list(set(photoinstrument))))
	instruments = ", ".join(instrulist)
	if len(instrulist) > 0:
		catalog['instruments'] = instruments

	catalogrows.append([catalog[x] for x in columnkey])

	tools = "pan,wheel_zoom,box_zoom,save,crosshair,hover,reset,resize"

	if plotavail:
		x_data = phototime

		x_buffer = 0.1*(max(x_data) - min(x_data))
		x_range = [-x_buffer + min(x_data), x_buffer + max(x_data)]

		p1 = figure(title='Photometry for ' + eventname, x_axis_label='Time (' + phototu[0] + ')',
			y_axis_label='AB Magnitude', x_range = x_range, tools = tools,
			y_range = (0.5 + max([x + y for x, y in zip(photoAB, photoerr)]), -0.5 + min([x - y for x, y in zip(photoAB, photoerr)])))

		err_xs = []
		err_ys = []

		for x, y, yerr in zip(phototime, photoAB, photoerr):
			err_xs.append((x, x))
			err_ys.append((y - yerr, y + yerr))

		bandset = set(photoband)
		bandset = [i for (j, i) in sorted(zip(map(bandnamef, bandset), bandset))]

		for band in bandset:
			bandname = bandnamef(band)
			indb = [i for i, j in enumerate(photoband) if j == band]
			indt = [i for i, j in enumerate(phototype) if j == 0]
			ind = set(indb).intersection(indt)

			p1.circle([phototime[x] for x in ind], [photoAB[x] for x in ind],
				color=bandcolorf(band), legend=bandname, size=5)
			p1.multi_line([err_xs[x] for x in ind], [err_ys[x] for x in ind], color=bandcolorf(band))

			upplimlegend = bandname if len(ind) == 0 else ''

			indt = [i for i, j in enumerate(phototype) if j == 1]
			ind = set(indb).intersection(indt)
			p1.inverted_triangle([phototime[x] for x in ind], [photoAB[x] for x in ind],
				color=bandcolorf(band), legend=upplimlegend, size=8)

		p = p1

		#save(p)
		html = file_html(p, CDN, eventname)
		returnlink = r'    << Return to supernova catalog';
		html = re.sub(r'(\)', r'\1\n    '+returnlink, html)
		html = re.sub(r'(\<\/body\>)', r'Download datafile

\n	  \1', html)
		html = re.sub(r'(\<\/body\>)', returnlink+r'\n	  \1', html)
		print outdir + eventname + ".html"
		with open(outdir + eventname + ".html", "w") as f:
			f.write(html)

# Write it all out at the end
csvout = open(outdir + 'sne-catalog.csv', 'wb')
csvout = csv.writer(csvout, quotechar='"', quoting=csv.QUOTE_ALL)

csvout.writerow(headerrow)

for row in catalogrows:
	csvout.writerow(row)

footer = dict(zip(columnkey,footer))
csvout.writerow([footer[x] for x in columnkey])
<commit_msg>Changed another .dat link to .dat.bz2<commit_after>#!/usr/local/bin/python2.7

import csv
import glob
import sys
import os
import re
import bz2
from bokeh.io import hplot, vplot
from bokeh.plotting import figure, show, save
from bokeh.resources import CDN
from bokeh.embed import file_html

indir = "../data/"
outdir = "../"

header = [
	"Names",
	"Host Names",
#	"Publications",
	"Instruments/Surveys",
	"z",
	r"vHelio",
#	"$N_{\\rm h}$",
	"Claimed Type",
#	"Notes",
	"Plots",
	"Data"
	]

columnkey = [
	"name",
	"host",
#	"citations",
	"instruments",
	"redshift",
	"hvel",
#	"nh",
	"claimedtype",
#	"notes",
	"plot",
	"data"
	]

footer = [
	"Note: IAU name preferred",
	"* Uncertain",
#	"* discovery\n‡ sne type first proposed",
	"",
	"",
	"",
#	"Line of sight H column",
	"",
#	"",
	"",
	""
	]

if (len(columnkey) != len(header) or len(columnkey) != len(footer)):
	print 'Error: Header and/or footer not same length as key list.'
	sys.exit(0)

dataavaillink = "Y";

header = dict(zip(columnkey,header))
headerrow = [header[x] for x in columnkey]

bandcolors = [
	"indigo",
	"firebrick",
	"forestgreen",
	"red",
	"crimson",
	"indigo",
	"darkblue",
	"mediumvioletred",
	"pink",
	"#d82930",
	"orangered",
	"mediumvioletred",
	"mediumspringgreen",
	"orange",
	"chocolate",
	"darkorange",
]

bandcodes = [
	"u",
	"g",
	"r",
	"i",
	"z",
	"U",
	"B",
	"V",
	"R",
	"I",
	"G",
	"Y",
	"J",
	"H",
	"K"
]

bandnames = [
	"u",
	"g",
	"r",
	"i",
	"z",
	"U",
	"B",
	"V",
	"R",
	"I",
	"G",
	"Y",
	"J",
	"H",
	"K"
]

bandcolordict = dict(zip(bandcodes,bandcolors))
bandnamedict = dict(zip(bandcodes,bandnames))

def bandcolorf(color):
	if (color in bandcolordict):
		return bandcolordict[color]
	else:
		return 'black'

def bandnamef(code):
	if (code in bandnamedict):
		return bandnamedict[code]
	else:
		return code

catalogrows = []
for file in (sorted(glob.glob(indir + "*.bz2"), key=lambda s: s.lower()) + sorted(glob.glob(indir + "*.dat"), key=lambda s: s.lower())):
	print file
	filehead, ext = os.path.splitext(file)
	if ext == ".dat":
		tsvin = open(file,'rb')
	elif ext == ".bz2":
		tsvin = bz2.BZ2File(file,'rb')
	else:
		print "illegal file extension"
	tsvin = csv.reader(tsvin, delimiter='\t')

	catalog = dict(zip(columnkey,['' for _ in xrange(len(columnkey))]))

	table = []

	phototu = []
	phototime = []
	photoband = []
	photoinstrument = []
	photoAB = []
	photoerr = []
	phototype = []

	eventname = os.path.basename(file).split('.')[0]

	plotavail = False;
	for row in tsvin:
		if row[0] == 'photometry':
			plotavail = True;
			plotlink = "";
			catalog['plot'] = plotlink

			phototu.append(row[1])
			phototime.append(float(row[2]))
			photoband.append(row[4])
			photoinstrument.append(row[6].strip())
			photoAB.append(float(row[8]))
			if not row[10]:
				photoerr.append(0.)
			else:
				photoerr.append(float(row[10]))
			phototype.append(int(row[12]))

		if row[0] in columnkey:
			table.append(row)
			catalog[row[0]] = row[1]

	catalog['data'] = r'Download'

	instrulist = sorted(filter(None, list(set(photoinstrument))))
	instruments = ", ".join(instrulist)
	if len(instrulist) > 0:
		catalog['instruments'] = instruments

	catalogrows.append([catalog[x] for x in columnkey])

	tools = "pan,wheel_zoom,box_zoom,save,crosshair,hover,reset,resize"

	if plotavail:
		x_data = phototime

		x_buffer = 0.1*(max(x_data) - min(x_data))
		x_range = [-x_buffer + min(x_data), x_buffer + max(x_data)]

		p1 = figure(title='Photometry for ' + eventname, x_axis_label='Time (' + phototu[0] + ')',
			y_axis_label='AB Magnitude', x_range = x_range, tools = tools,
			y_range = (0.5 + max([x + y for x, y in zip(photoAB, photoerr)]), -0.5 + min([x - y for x, y in zip(photoAB, photoerr)])))

		err_xs = []
		err_ys = []

		for x, y, yerr in zip(phototime, photoAB, photoerr):
			err_xs.append((x, x))
			err_ys.append((y - yerr, y + yerr))

		bandset = set(photoband)
		bandset = [i for (j, i) in sorted(zip(map(bandnamef, bandset), bandset))]

		for band in bandset:
			bandname = bandnamef(band)
			indb = [i for i, j in enumerate(photoband) if j == band]
			indt = [i for i, j in enumerate(phototype) if j == 0]
			ind = set(indb).intersection(indt)

			p1.circle([phototime[x] for x in ind], [photoAB[x] for x in ind],
				color=bandcolorf(band), legend=bandname, size=5)
			p1.multi_line([err_xs[x] for x in ind], [err_ys[x] for x in ind], color=bandcolorf(band))

			upplimlegend = bandname if len(ind) == 0 else ''

			indt = [i for i, j in enumerate(phototype) if j == 1]
			ind = set(indb).intersection(indt)
			p1.inverted_triangle([phototime[x] for x in ind], [photoAB[x] for x in ind],
				color=bandcolorf(band), legend=upplimlegend, size=8)

		p = p1

		#save(p)
		html = file_html(p, CDN, eventname)
		returnlink = r'    << Return to supernova catalog';
		html = re.sub(r'(\)', r'\1\n    '+returnlink, html)
		html = re.sub(r'(\<\/body\>)', r'Download datafile

\n	  \1', html)
		html = re.sub(r'(\<\/body\>)', returnlink+r'\n	  \1', html)
		print outdir + eventname + ".html"
		with open(outdir + eventname + ".html", "w") as f:
			f.write(html)

# Write it all out at the end
csvout = open(outdir + 'sne-catalog.csv', 'wb')
csvout = csv.writer(csvout, quotechar='"', quoting=csv.QUOTE_ALL)

csvout.writerow(headerrow)

for row in catalogrows:
	csvout.writerow(row)

footer = dict(zip(columnkey,footer))
csvout.writerow([footer[x] for x in columnkey])
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""The report builders are classes that coordinate report productions"""

import warnings
import numpy as np

import scipy.spatial.distance as dist
import scipy.cluster.hierarchy as hier

class ReportBuilderBase(object):
    """Base class for common report builder features.

    Most prominently, the distill-method needs to be implemented in
    subclasses to make much sence in most use cases.

    Attributes
    ----------

    outputRoot
    outputNamePrefix

    """
    def __init__(self, outputRoot=None, outputNamePrefix=None, *reports):
        """
        Parameters
        ----------

        outputRoot: str, optional
            Path to the directory where all reports should be put

        outputNamePrefix: str, optional
            Partial name to be added to all reports done by the builder

        *reports: objects, optional
            Any number of reports to be added from start
        """

        self._reports = set()
        self.outputRoot=outputRoot
        self.outputNamePrefix=outputNamePrefix
        self.addReports(*reports)

    @property
    def outputNamePrefix(self):
        """Partial file name to prepend the individual reports: str"""

        return self._outputNamePrefix

    @outputNamePrefix.setter
    def outputNamePrefix(self, val):

        self._outputNamePrefix = val

    @property
    def outputRoot(self):
        """The base for saving out reports: str"""

        return self._outputRoot

    @outputRoot.setter
    def outputRoot(self, val):

        if val is None:
            val = ''

        self._outputRoot = str(val)

    def addReports(self, *reports):
        """Adds any number of reports given that the reports exposes a
        distill method.

        Parameters
        ----------

        *reports: objects, optional

        Returns
        -------

        fseq.ReportBuilderBase
            Returns `self`

        Raises
        ------

        ValueError
            If a report lacks a method named `distill`
        """

        for r in reports:

            if r not in self._reports:
                if hasattr(r, 'distill'):
                    self._reports.add(r)
                else:
                    raise ValueError("{0} lacks distill-method".format(r))
            else:
                warnings.warn("{0} already in reports...omitting.".format(r))

        return self

    def distill(self, *args, **kwargs):
        """The base distiller will not process any data passed to it,
        it will send all arguments and keyword arguments to the individual
        reports.

        If either `outputRoot` or `outputNamePrefix` are passed as kwargs,
        the corresponding values preset in the system will be added to the
        kwargs sent to the subreports.

        Returns
        -------

        fseq.ReportBuilderBase
            Returns `self`
        """
        for k in ('outputRoot', 'outputNamePrefix'):
            if k not in kwargs:
                kwargs[k] = getattr(self, k)

        for r in self._reports:
            r.distill(*args, **kwargs)

        return self

class ReportBuilderFFT(ReportBuilderBase):
    """Samples part of data set and performs FFT-based analysis on it.

    Attributes
    ----------

    distanceMetric
    sampleSize

    See also
    --------

    ReportBuilderBase
        Base class implementing more attributes.
    """

    METRICS = {'braycurtis', 'canberra', 'chebyshev', 'cityblock',
               'correlation', 'cosine', 'dice', 'euclidean', 'hamming',
               'jaccard', 'kulsinski', 'mahalanobis', 'matching',
               'minkowski', 'rogerstanimoto', 'russellrao', 'seuclidean',
               'sokalmichener', 'sokalsneath', 'sqeuclidean', 'yule'}

    def __init__(self, outputRoot=None, outputNamePrefix=None,
                 sampleSize=1000, distanceMetric='correlation', *reports):
        """
        Parameters
        ----------

        outputRoot: str, optional
            Path to the directory where all reports should be put

        outputNamePrefix: str, optional
            Partial name to be added to all reports done by the builder

        sampleSize: int, optional
            Size of sample to be randomly drawn (Default: 1000)

        distanceMetric: str, optional
            Name of distance metric to be used.
            See `ReportBuilderFFT.METRICS` for allowed metrics.
            (Default: 'correlation')

        *reports: objects, optional
            Any number of reports to be added from start
        """
        super(ReportBuilderFFT, self).__init__(
            outputRoot=outputRoot, outputNamePrefix=outputNamePrefix,
            *reports)

        self.sampleSize = sampleSize
        self.distanceMetric = distanceMetric

    @property
    def distanceMetric(self):

        return self._distanceMetric

    @distanceMetric.setter
    def distanceMetric(self, metric):

        metric = metric.lower()
        if metric in self.METRICS:
            self._distanceMetric = metric
        else:
            raise ValueError("{0} not a valid metric ({1})".format(
                metric, self.METRICS))

    @property
    def sampleSize(self):

        return self._sampleSize

    @sampleSize.setter
    def sampleSize(self, val):

        self._sampleSize = val

    def _getLeafOrder(self, A, metric, w=None, V=None, VI=None):

        distMatrix = dist.pdist(A, metric=metric, w=w, V=V, VI=VI)

        distSquareM = dist.squareform(distMatrix)

        linkageM = hier.linkage(distSquareM)

        dendro = hier.dendrogram(linkageM)

        return dendro['leaves']

    def distill(self, data, distanceMetric=None, clusterOnAbsOnly=True,
            *args, **kwargs):
        """Make reports from data.

        Produces two analyses:

        Amplitude evaluation
            A clustered FFT-amplitude analysis

        Angle evaluation
            A clustered FFT-angle analysis

        Parameters
        ----------

        data: numpy.ndarray
            The 2D-array of data given

        distanceMetric: str, optional
            A distance metric to overwrite the default one of the instance.

        clusterOnAbsOnly: bool, optional
            If clustering should be performed only on the amplitude (abs-values)
            or if amplitude and angle be clustered independently.
            (Default: Cluster only on amplitude)
        """
        if distanceMetric is None:
            distanceMetric = self.distanceMetric

        D = np.arange(data.shape[0])
        np.random.shuffle(D)
        data = data[D[:self.sampleSize]]

        fD = np.fft.rfft(data, axis=1)

        A = np.abs(fD)
        V = np.std(A, axis=0)
        O = self._getLeafOrder(A, distanceMetric, V=V)

        super(ReportBuilderFFT, self).distill(
            A[O],
            outputNamePrefix='fft-sample.abs.',
            title='absolute FFT values for {0} random sample sequences'.format(
                self.sampleSize),
            xlabel='Frequency',
            ylabel='Read n',
            axisOff=False,
            *args, **kwargs)

        #A = (np.angle(fD) - np.angle(fD[:, :1])) % (2 * np.pi)
        A = np.angle(fD)
        V = np.std(A, axis=0)
        if not clusterOnAbsOnly:
            O = self._getLeafOrder(A, distanceMetric, V=V)

        super(ReportBuilderFFT, self).distill(
            A[O],
            outputNamePrefix='fft-sample.angle.',
            title='FFT angle for {0} random sample sequences'.format(
                self.sampleSize),
            xlabel='Frequency',
            ylabel='Read n',
            axisOff=False,
            *args, **kwargs)

class ReportBuilderPositionAverage(ReportBuilderBase):
    """Per position analysis builder.

    Attributes
    ----------

    undecidedValue

    See also
    --------

    ReportBuilderBase
        Base class which implements some more attributes.
    """
    def __init__(self, outputRoot=None, outputNamePrefix=None,
                 undecidedValue=0.5, *reports):
        """
        Parameters
        ----------

        outputRoot: str, optional
            Path to the directory where all reports should be put

        outputNamePrefix: str, optional
            Partial name to be added to all reports done by the builder

        undecidedValue: int, optional
            The value for which undecided items were encoded (so it
            can be omited and calculated separately for 2 of 3 graphs).
            (Default: 0.5)

        *reports: objects, optional
            Any number of reports to be added from start
        """

        super(ReportBuilderPositionAverage, self).__init__(
            outputRoot=outputRoot, outputNamePrefix=outputNamePrefix,
            *reports)

        self.undecidedValue = undecidedValue

    @property
    def undecidedValue(self):
        return self._undecidedValue

    @undecidedValue.setter
    def undecidedValue(self, val):
        self._undecidedValue = val

    def _floatBin(self, C):

        u = np.unique(C)
        return np.histogram(C, bins=np.hstack((np.unique(C), (np.inf,))))

    def _getF(self, X, Y, val):

        if val not in Y:
            return 0

        return X[Y[:X.size] == val] / X.sum().astype(np.float)

    def _getNotF(self, X, Y, val):

        pos = Y[:X.size] != val
        return (X[pos] * Y[pos]).sum() / X.sum().astype(np.float)

    def distill(self, data, undecidedValue=None, *args, **kwargs):
        """The distiller will create reports for several position-type
        informations.

        Average Lacking Data Frequency
            The number of undecided values.

        Average Non-Lacking Data
            The per position average for all non-lacking data values.

        Average Combined Data
            The per position average as encoded.

        Parameters
        ----------

        data: numpy.ndarray
            An array of numerically encoded sequence information

        undecidedValue: float, optional
            The value that undecided sequence positions are encoded as.
            If not supplied, the previously set value of the class instance
            will be used.
            (Default: 0.5)

        *args:
            Any args will be passed to the `ReportBuilderBase.distill`

        **kwargs:
            Any kwargs will be passed to the `ReportBuilderBase.distill`
            ..note:: `outputNamePrefix` will be overwritten/added

        Returns
        -------

        fseq.ReportBuilderPositionAverage
            Returns `self`
        """

        if undecidedValue is None:
            undecidedValue = self.undecidedValue

        super(ReportBuilderPositionAverage, self).distill(
            data.mean(axis=0, dtype=np.float),
            outputNamePrefix='average.total.',
            title='Average GC per position',
            xlabel='Read position',
            ylabel='%GC',
            *args, **kwargs)

        freqs, bins = zip(*(self._floatBin(C) for C in data.T))
        undecidedV = np.ones_like(freqs) * undecidedValue

        lacking = np.frompyfunc(self._getF, 3, 1)

        super(ReportBuilderPositionAverage, self).distill(
            lacking(freqs, bins, undecidedV),
            title='Frequency of missing data per position',
            xlabel='Read position',
            ylabel='f',
            outputNamePrefix='average.lacking.', *args, **kwargs)

        notLacking = np.frompyfunc(self._getNotF, 3, 1)

        super(ReportBuilderPositionAverage, self).distill(
            notLacking(freqs, bins, undecidedV),
            title='Average GC per position, omitting uncertain',
            xlabel='Read position',
            ylabel='%GC',
            outputNamePrefix='average.not-lacking.', *args, **kwargs)

        return self
<commit_msg>FEAT: Added ability to iterate through the report builders reports<commit_after>#!/usr/bin/env python
"""The report builders are classes that coordinate report productions"""

import warnings
import numpy as np

import scipy.spatial.distance as dist
import scipy.cluster.hierarchy as hier

class ReportBuilderBase(object):
    """Base class for common report builder features.

    Most prominently, the distill-method needs to be implemented in
    subclasses to make much sence in most use cases.

    Attributes
    ----------

    outputRoot
    outputNamePrefix

    """
    def __init__(self, outputRoot=None, outputNamePrefix=None, *reports):
        """
        Parameters
        ----------

        outputRoot: str, optional
            Path to the directory where all reports should be put

        outputNamePrefix: str, optional
            Partial name to be added to all reports done by the builder

        *reports: objects, optional
            Any number of reports to be added from start
        """

        self._reports = set()
        self.outputRoot=outputRoot
        self.outputNamePrefix=outputNamePrefix
        self.addReports(*reports)

    def __iter__(self):

        return iter(self._reports)

    @property
    def outputNamePrefix(self):
        """Partial file name to prepend the individual reports: str"""

        return self._outputNamePrefix

    @outputNamePrefix.setter
    def outputNamePrefix(self, val):

        self._outputNamePrefix = val

    @property
    def outputRoot(self):
        """The base for saving out reports: str"""

        return self._outputRoot

    @outputRoot.setter
    def outputRoot(self, val):

        if val is None:
            val = ''

        self._outputRoot = str(val)

    def addReports(self, *reports):
        """Adds any number of reports given that the reports exposes a
        distill method.

        Parameters
        ----------

        *reports: objects, optional

        Returns
        -------

        fseq.ReportBuilderBase
            Returns `self`

        Raises
        ------

        ValueError
            If a report lacks a method named `distill`
        """

        for r in reports:

            if r not in self._reports:
                if hasattr(r, 'distill'):
                    self._reports.add(r)
                else:
                    raise ValueError("{0} lacks distill-method".format(r))
            else:
                warnings.warn("{0} already in reports...omitting.".format(r))

        return self

    def distill(self, *args, **kwargs):
        """The base distiller will not process any data passed to it,
        it will send all arguments and keyword arguments to the individual
        reports.

        If either `outputRoot` or `outputNamePrefix` are passed as kwargs,
        the corresponding values preset in the system will be added to the
        kwargs sent to the subreports.

        Returns
        -------

        fseq.ReportBuilderBase
            Returns `self`
        """
        for k in ('outputRoot', 'outputNamePrefix'):
            if k not in kwargs:
                kwargs[k] = getattr(self, k)

        for r in self._reports:
            r.distill(*args, **kwargs)

        return self

class ReportBuilderFFT(ReportBuilderBase):
    """Samples part of data set and performs FFT-based analysis on it.

    Attributes
    ----------

    distanceMetric
    sampleSize

    See also
    --------

    ReportBuilderBase
        Base class implementing more attributes.
    """

    METRICS = {'braycurtis', 'canberra', 'chebyshev', 'cityblock',
               'correlation', 'cosine', 'dice', 'euclidean', 'hamming',
               'jaccard', 'kulsinski', 'mahalanobis', 'matching',
               'minkowski', 'rogerstanimoto', 'russellrao', 'seuclidean',
               'sokalmichener', 'sokalsneath', 'sqeuclidean', 'yule'}

    def __init__(self, outputRoot=None, outputNamePrefix=None,
                 sampleSize=1000, distanceMetric='correlation', *reports):
        """
        Parameters
        ----------

        outputRoot: str, optional
            Path to the directory where all reports should be put

        outputNamePrefix: str, optional
            Partial name to be added to all reports done by the builder

        sampleSize: int, optional
            Size of sample to be randomly drawn (Default: 1000)

        distanceMetric: str, optional
            Name of distance metric to be used.
            See `ReportBuilderFFT.METRICS` for allowed metrics.
            (Default: 'correlation')

        *reports: objects, optional
            Any number of reports to be added from start
        """
        super(ReportBuilderFFT, self).__init__(
            outputRoot=outputRoot, outputNamePrefix=outputNamePrefix,
            *reports)

        self.sampleSize = sampleSize
        self.distanceMetric = distanceMetric

    @property
    def distanceMetric(self):

        return self._distanceMetric

    @distanceMetric.setter
    def distanceMetric(self, metric):

        metric = metric.lower()
        if metric in self.METRICS:
            self._distanceMetric = metric
        else:
            raise ValueError("{0} not a valid metric ({1})".format(
                metric, self.METRICS))

    @property
    def sampleSize(self):

        return self._sampleSize

    @sampleSize.setter
    def sampleSize(self, val):

        self._sampleSize = val

    def _getLeafOrder(self, A, metric, w=None, V=None, VI=None):

        distMatrix = dist.pdist(A, metric=metric, w=w, V=V, VI=VI)

        distSquareM = dist.squareform(distMatrix)

        linkageM = hier.linkage(distSquareM)

        dendro = hier.dendrogram(linkageM)

        return dendro['leaves']

    def distill(self, data, distanceMetric=None, clusterOnAbsOnly=True,
            *args, **kwargs):
        """Make reports from data.

        Produces two analyses:

        Amplitude evaluation
            A clustered FFT-amplitude analysis

        Angle evaluation
            A clustered FFT-angle analysis

        Parameters
        ----------

        data: numpy.ndarray
            The 2D-array of data given

        distanceMetric: str, optional
            A distance metric to overwrite the default one of the instance.

        clusterOnAbsOnly: bool, optional
            If clustering should be performed only on the amplitude (abs-values)
            or if amplitude and angle be clustered independently.
            (Default: Cluster only on amplitude)
        """
        if distanceMetric is None:
            distanceMetric = self.distanceMetric

        D = np.arange(data.shape[0])
        np.random.shuffle(D)
        data = data[D[:self.sampleSize]]

        fD = np.fft.rfft(data, axis=1)

        A = np.abs(fD)
        V = np.std(A, axis=0)
        O = self._getLeafOrder(A, distanceMetric, V=V)

        super(ReportBuilderFFT, self).distill(
            A[O],
            outputNamePrefix='fft-sample.abs.',
            title='absolute FFT values for {0} random sample sequences'.format(
                self.sampleSize),
            xlabel='Frequency',
            ylabel='Read n',
            axisOff=False,
            *args, **kwargs)

        #A = (np.angle(fD) - np.angle(fD[:, :1])) % (2 * np.pi)
        A = np.angle(fD)
        V = np.std(A, axis=0)
        if not clusterOnAbsOnly:
            O = self._getLeafOrder(A, distanceMetric, V=V)

        super(ReportBuilderFFT, self).distill(
            A[O],
            outputNamePrefix='fft-sample.angle.',
            title='FFT angle for {0} random sample sequences'.format(
                self.sampleSize),
            xlabel='Frequency',
            ylabel='Read n',
            axisOff=False,
            *args, **kwargs)

class ReportBuilderPositionAverage(ReportBuilderBase):
    """Per position analysis builder.

    Attributes
    ----------

    undecidedValue

    See also
    --------

    ReportBuilderBase
        Base class which implements some more attributes.
    """
    def __init__(self, outputRoot=None, outputNamePrefix=None,
                 undecidedValue=0.5, *reports):
        """
        Parameters
        ----------

        outputRoot: str, optional
            Path to the directory where all reports should be put

        outputNamePrefix: str, optional
            Partial name to be added to all reports done by the builder

        undecidedValue: int, optional
            The value for which undecided items were encoded (so it
            can be omited and calculated separately for 2 of 3 graphs).
            (Default: 0.5)

        *reports: objects, optional
            Any number of reports to be added from start
        """

        super(ReportBuilderPositionAverage, self).__init__(
            outputRoot=outputRoot, outputNamePrefix=outputNamePrefix,
            *reports)

        self.undecidedValue = undecidedValue

    @property
    def undecidedValue(self):
        return self._undecidedValue

    @undecidedValue.setter
    def undecidedValue(self, val):
        self._undecidedValue = val

    def _floatBin(self, C):

        u = np.unique(C)
        return np.histogram(C, bins=np.hstack((np.unique(C), (np.inf,))))

    def _getF(self, X, Y, val):

        if val not in Y:
            return 0

        return X[Y[:X.size] == val] / X.sum().astype(np.float)

    def _getNotF(self, X, Y, val):

        pos = Y[:X.size] != val
        return (X[pos] * Y[pos]).sum() / X.sum().astype(np.float)

    def distill(self, data, undecidedValue=None, *args, **kwargs):
        """The distiller will create reports for several position-type
        informations.

        Average Lacking Data Frequency
            The number of undecided values.

        Average Non-Lacking Data
            The per position average for all non-lacking data values.

        Average Combined Data
            The per position average as encoded.

        Parameters
        ----------

        data: numpy.ndarray
            An array of numerically encoded sequence information

        undecidedValue: float, optional
            The value that undecided sequence positions are encoded as.
            If not supplied, the previously set value of the class instance
            will be used.
            (Default: 0.5)

        *args:
            Any args will be passed to the `ReportBuilderBase.distill`

        **kwargs:
            Any kwargs will be passed to the `ReportBuilderBase.distill`
            ..note:: `outputNamePrefix` will be overwritten/added

        Returns
        -------

        fseq.ReportBuilderPositionAverage
            Returns `self`
        """

        if undecidedValue is None:
            undecidedValue = self.undecidedValue

        super(ReportBuilderPositionAverage, self).distill(
            data.mean(axis=0, dtype=np.float),
            outputNamePrefix='average.total.',
            title='Average GC per position',
            xlabel='Read position',
            ylabel='%GC',
            *args, **kwargs)

        freqs, bins = zip(*(self._floatBin(C) for C in data.T))
        undecidedV = np.ones_like(freqs) * undecidedValue

        lacking = np.frompyfunc(self._getF, 3, 1)

        super(ReportBuilderPositionAverage, self).distill(
            lacking(freqs, bins, undecidedV),
            title='Frequency of missing data per position',
            xlabel='Read position',
            ylabel='f',
            outputNamePrefix='average.lacking.', *args, **kwargs)

        notLacking = np.frompyfunc(self._getNotF, 3, 1)

        super(ReportBuilderPositionAverage, self).distill(
            notLacking(freqs, bins, undecidedV),
            title='Average GC per position, omitting uncertain',
            xlabel='Read position',
            ylabel='%GC',
            outputNamePrefix='average.not-lacking.', *args, **kwargs)

        return self
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Program: nicerfits2bin.py
# Authors: Paul S. Ray <paul.ray@nrl.navy.mil>
# Description:
# Reads a NICER clean FITS file of photon event times
# and generates a .bin and .inf file suitable for PRESTO analysis
# The .bin is just a binary file of double precision MJD times
from __future__ import division, print_function
import os,sys
import argparse
import numpy as np
from astropy import log
import astropy.units as u
import astropy.io.fits as pyfits
from astropy.time import Time
from astropy.table import Table
from nicer.values import *
from nicer.plotutils import choose_N
from subprocess import check_call

def runcmd(cmd):
    # CMD should be a list of strings since it is not processed by a shell
    log.info('CMD: '+" ".join(cmd))
    #log.info(cmd)
    check_call(cmd,env=os.environ)

desc="""Convert NICER events to PRESTO bin+inf files. """

parser = argparse.ArgumentParser(description=desc)
parser.add_argument("evfile", help="Name of event file to convert (must be barycentered)")
parser.add_argument("--dt", help="Time series bin size", type=float, default=1.0/128.0)
parser.add_argument("--ra",help="RA in HH:MM:SS.s",default="00:00:00")
parser.add_argument("--dec",help="DEC in DD:MM:SS.s", default="00:00:00")

parser.add_argument("--observer",help="Name of person analyzing data",default=None)
parser.add_argument("--force",help="Force processing even if not barycentered",default=False,action="store_true")
parser.add_argument("--search",help="Do an FFT and search the data after creating it",default=False,action="store_true")
parser.add_argument("--numharm",help="Number of harmonics to sum in search",default="2")
parser.add_argument("--zmax",help="Max z for acceleration search",default="50")
parser.add_argument("--flo",help="Lowest freq (of highest harmonic) to search",default="0.02")
args = parser.parse_args()
if args.observer is None:
    import getpass
    args.observer = getpass.getuser()

# Grab base filename for output
base = os.path.splitext(os.path.basename(args.evfile))[0]

etable = Table.read(args.evfile,hdu=1)
if etable.meta['TIMESYS'] != 'TDB' and not args.force:
    log.error('Event file must be barycentered!')
    sys.exit(1)
gtitable = Table.read(args.evfile,hdu=2)

epoch_met = gtitable['START'][0]
# WARNING: This loses precision! Should be done with astropy times
# Should make utility routine to convert FITS TIME column to astropy times properly
epoch_mjd = (etable.meta['MJDREFI'] + etable.meta['MJDREFF']
    + etable.meta['TIMEZERO'] + epoch_met/86400.0)

# Write event times to bin file
eventtimes = np.array(etable['TIME'],dtype=np.float)-epoch_met
log.info('Event times: {0} to {1}'.format(eventtimes.min(),eventtimes.max()))
eventtimes.tofile('{0}.events'.format(base))
#
nbins = choose_N((gtitable['STOP'][-1]-epoch_met)/args.dt)
log.info('Using {0} bins of width {1} seconds for total of {2} s'.format(nbins,
    args.dt,nbins*args.dt))

# Now write out .inf file
inf = open('%s.inf' % base,'w')
infstr = """ Data file name without suffix          =  {0}
 Telescope used                         =  NICER
 Instrument used                        =  XTI
 Object being observed                  =  {1}
 J2000 Right Ascension (hh:mm:ss.ssss)  =  {2}
 J2000 Declination     (dd:mm:ss.ssss)  =  {3}
 Data observed by                       =  NICER
 Epoch of observation (MJD)             =  {4:.9f}
 Barycentered?           (1=yes, 0=no)  =  1
 Number of bins in the time series      =  {5}
 Width of each time series bin (sec)    =  {6}
 Any breaks in the data? (1=yes, 0=no)  =  0
 Type of observation (EM band)          =  X-ray
 Field-of-view diameter (arcsec)        =  180.00
 Central energy (kev)                   =  4.2
 Energy bandpass (kev)                  =  3.8
 Data analyzed by                       =  {7}
 Any additional notes:
    None
""".format(base,etable.meta['OBJECT'], args.ra, args.dec, epoch_mjd, nbins,
    args.dt,args.observer)

inf.write(infstr)
inf.close()

# Now bin data and write .dat file
bins = np.arange(nbins+1,dtype=np.float)*args.dt
sums, edges = np.histogram(eventtimes, bins=bins)
dat = np.array(sums,np.float32)
dat.tofile('{0}.dat'.format(base))

if args.search:
    cmd = ['realfft', '{0}.dat'.format(base)]
    runcmd(cmd)
    cmd = ['accelsearch', '-zmax', args.zmax, '-flo',args.flo, '-numharm', args.numharm,'-sigma','1.5','-photon','{0}.fft'.format(base)]
    runcmd(cmd)
    # If candidates are found, fold the most significant one
    if os.path.isfile('{0}_ACCEL_{1}.cand'.format(base,args.zmax)):
        cmd = ['prepfold', '-noxwin', '-events', '-double', '-accelcand', '1',
            '-accelfile', '{0}_ACCEL_{1}.cand'.format(base,args.zmax),
            '{0}.events'.format(base)]
        runcmd(cmd)
<commit_msg>Tiny bug fixx<commit_after>#!/usr/bin/env python
# Program: nicerfits2bin.py
# Authors: Paul S. Ray <paul.ray@nrl.navy.mil>
# Description:
# Reads a NICER clean FITS file of photon event times
# and generates a .bin and .inf file suitable for PRESTO analysis
# The .bin is just a binary file of double precision MJD times
from __future__ import division, print_function
import os,sys
import argparse
import numpy as np
from astropy import log
import astropy.units as u
import astropy.io.fits as pyfits
from astropy.time import Time
from astropy.table import Table
from nicer.values import *
from nicer.plotutils import choose_N
from subprocess import check_call

def runcmd(cmd):
    # CMD should be a list of strings since it is not processed by a shell
    log.info('CMD: '+" ".join(cmd))
    #log.info(cmd)
    check_call(cmd,env=os.environ)

desc="""Convert NICER events to PRESTO bin+inf files. """

parser = argparse.ArgumentParser(description=desc)
parser.add_argument("evfile", help="Name of event file to convert (must be barycentered)")
parser.add_argument("--dt", help="Time series bin size", type=float, default=1.0/128.0)
parser.add_argument("--ra",help="RA in HH:MM:SS.s",default="00:00:00")
parser.add_argument("--dec",help="DEC in DD:MM:SS.s", default="00:00:00")

parser.add_argument("--observer",help="Name of person analyzing data",default=None)
parser.add_argument("--force",help="Force processing even if not barycentered",default=False,action="store_true")
parser.add_argument("--search",help="Do an FFT and search the data after creating it",default=False,action="store_true")
parser.add_argument("--numharm",help="Number of harmonics to sum in search",default="2")
parser.add_argument("--zmax",help="Max z for acceleration search",default="50")
parser.add_argument("--flo",help="Lowest freq (of highest harmonic) to search",default="0.02")
args = parser.parse_args()
if args.observer is None:
    import getpass
    args.observer = getpass.getuser()

# Grab base filename for output
base = os.path.splitext(os.path.basename(args.evfile))[0]

etable = Table.read(args.evfile,hdu=1)
if etable.meta['TIMESYS'] != 'TDB' and not args.force:
    log.error('Event file must be barycentered!')
    sys.exit(1)
gtitable = Table.read(args.evfile,hdu=2)

epoch_met = gtitable['START'][0]
# WARNING: This loses precision! Should be done with astropy times
# Should make utility routine to convert FITS TIME column to astropy times properly
epoch_mjd = (etable.meta['MJDREFI'] + etable.meta['MJDREFF']
    + (etable.meta['TIMEZERO'] + epoch_met)/86400.0)

# Write event times to bin file
eventtimes = np.array(etable['TIME'],dtype=np.float)-epoch_met
log.info('Event times: {0} to {1}'.format(eventtimes.min(),eventtimes.max()))
eventtimes.tofile('{0}.events'.format(base))
#
nbins = choose_N((gtitable['STOP'][-1]-epoch_met)/args.dt)
log.info('Using {0} bins of width {1} seconds for total of {2} s'.format(nbins,
    args.dt,nbins*args.dt))

# Now write out .inf file
inf = open('%s.inf' % base,'w')
infstr = """ Data file name without suffix          =  {0}
 Telescope used                         =  NICER
 Instrument used                        =  XTI
 Object being observed                  =  {1}
 J2000 Right Ascension (hh:mm:ss.ssss)  =  {2}
 J2000 Declination     (dd:mm:ss.ssss)  =  {3}
 Data observed by                       =  NICER
 Epoch of observation (MJD)             =  {4:.9f}
 Barycentered?           (1=yes, 0=no)  =  1
 Number of bins in the time series      =  {5}
 Width of each time series bin (sec)    =  {6}
 Any breaks in the data? (1=yes, 0=no)  =  0
 Type of observation (EM band)          =  X-ray
 Field-of-view diameter (arcsec)        =  180.00
 Central energy (kev)                   =  4.2
 Energy bandpass (kev)                  =  3.8
 Data analyzed by                       =  {7}
 Any additional notes:
    None
""".format(base,etable.meta['OBJECT'], args.ra, args.dec, epoch_mjd, nbins,
    args.dt,args.observer)

inf.write(infstr)
inf.close()

# Now bin data and write .dat file
bins = np.arange(nbins+1,dtype=np.float)*args.dt
sums, edges = np.histogram(eventtimes, bins=bins)
dat = np.array(sums,np.float32)
dat.tofile('{0}.dat'.format(base))

if args.search:
    cmd = ['realfft', '{0}.dat'.format(base)]
    runcmd(cmd)
    cmd = ['accelsearch', '-zmax', args.zmax, '-flo',args.flo, '-numharm', args.numharm,'-sigma','1.5','-photon','{0}.fft'.format(base)]
    runcmd(cmd)
    # If candidates are found, fold the most significant one
    if os.path.isfile('{0}_ACCEL_{1}.cand'.format(base,args.zmax)):
        cmd = ['prepfold', '-noxwin', '-events', '-double', '-accelcand', '1',
            '-accelfile', '{0}_ACCEL_{1}.cand'.format(base,args.zmax),
            '{0}.events'.format(base)]
        runcmd(cmd)
<|endoftext|>


<commit_before><commit_msg>Initialize Q2-Rabbits Multiplying<commit_after><|endoftext|>


<commit_before><commit_msg>pretty contour for L1 and cond phase<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys, os, re

sys.path.append("../../../utils/")
from segmaker import segmaker

tags = dict()

print("Preload all tags.")
for arg in sys.argv[1:]:
    with open(arg + ".txt", "r") as f:
        for line in f:
            tile, pip = line.split()
            _, pip = pip.split("/")
            tile_type, pip = pip.split(".")
            src, dst = pip.split("->>")
            tag = "%s.%s" % (dst, src)
            tags[tag] = dst

for arg in sys.argv[1:]:
    print("Processing %s." % arg)
    segmk = segmaker(arg + ".bits")

    tiledata = dict()
    pipdata = dict()
    ignpip = set()

    with open(arg + ".txt", "r") as f:
        for line in f:
            tile, pip = line.split()
            _, pip = pip.split("/")
            tile_type, pip = pip.split(".")
            src, dst = pip.split("->>")
            tag = "%s.%s" % (dst, src)
            segmk.addtag(tile, tag, 1)
            for tag, tag_dst in tags.items():
                if tag_dst != dst:
                    segmk.addtag(tile, tag, 0)

    segmk.compile()
    segmk.write(arg)

<commit_msg>Setup future 058-hclkpips improvements<commit_after>#!/usr/bin/env python3

import sys, os, re

sys.path.append("../../../utils/")
from segmaker import segmaker

tags = dict()

print("Preload all tags.")
for arg in sys.argv[1:]:
    with open(arg + ".txt", "r") as f:
        for line in f:
            tile, pip = line.split()
            _, pip = pip.split("/")
            tile_type, pip = pip.split(".")
            src, dst = pip.split("->>")
            tag = "%s.%s" % (dst, src)
            tags[tag] = (dst, src)

for arg in sys.argv[1:]:
    print("Processing %s." % arg)
    segmk = segmaker(arg + ".bits")

    tiledata = dict()
    pipdata = dict()
    ignpip = set()

    with open(arg + ".txt", "r") as f:
        for line in f:
            tile, pip = line.split()
            _, pip = pip.split("/")
            tile_type, pip = pip.split(".")
            src, dst = pip.split("->>")
            tag = "%s.%s" % (dst, src)
            segmk.addtag(tile, tag, 1)
            for tag, tag_dst_src in tags.items():
                tag_dst, tag_src = tag_dst_src
                if tag_dst != dst and (tag_src != src or True):
                    segmk.addtag(tile, tag, 0)

    segmk.compile()
    segmk.write(arg)

<|endoftext|>


<commit_before>
import logging

logger = logging.getLogger(__name__)

def initLayer(layer):
    pass

def initJob(job):
    pass

def setup():
    pass
<commit_msg>Finished plow output plugin, which handles after exec event on SetupTasks and registers paths with plow.<commit_after>"""
Plow outputs plugin.

Registers outputs with the plow server.
"""
import os
import logging
logger = logging.getLogger(__name__)

import blueprint

def afterExecute(layer):

    import plow

    # When a setup task is complete outputs should
    # be registerd with its parent layer.
    if not isinstance(layer, (blueprint.SetupTask,)):
        return

    parent = layer.getLayer()
    logger.info("Registering %d outputs" % len(parent.getOutputs()))
    for output in parent.getOutputs():
        logger.info("Registering output with plow: %s" % output.path)
        plow_layer = plow.getLayer(
            os.environ.get("PLOW_JOB_ID"), parent.getName())
        plow.registerOutput(plow_layer, output.path, {})

<|endoftext|>