<commit_before>#!/home/users/dnfisher/soft/virtual_envs/ggf/bin/python2

'''
Detects hotspot locations in a given file.  Used later
to detect flares based on hotspot persistency.
'''

import sys
import os
import logging
import zipfile
import shutil
import glob

import numpy as np
from netCDF4 import Dataset
import pandas as pd
from scipy.interpolate import RectBivariateSpline

import src.config.constants as proc_const

def extract_zip(input_zip, path_to_temp):
    data_dict = {}
    to_extract = ["S5_radiance_an.nc", "S6_radiance_an.nc",
                  "geodetic_an.nc", "geometry_tn.nc",
                  "cartesian_an.nc", "cartesian_tx.nc",
                  "indices_an.nc", "flags_an.nc"]
    with zipfile.ZipFile(input_zip) as input:
        for name in input.namelist():
            split_name = name.split('/')[-1]
            if split_name in to_extract:
                var_name = split_name.split('.')[0]
                source = Dataset(input.extract(name, path_to_temp))
                data_dict[var_name] = source

    # remove the unzip files
    dir_to_remove = os.path.join(path_to_temp, input_zip.split('/')[-1].replace('zip', 'SEN3'))
    if os.path.isdir(dir_to_remove):  # test if the path points to a directory
        shutil.rmtree(dir_to_remove, ignore_errors=True)
    else:  # normal file
        os.remove(dir_to_remove)

    return data_dict

def interpolate_szn(s3_data):
    szn = s3_data['geometry_tn']['solar_zenith_tn'][:]

    tx_x_var = s3_data['cartesian_tx']['x_tx'][0, :]
    tx_y_var = s3_data['cartesian_tx']['y_tx'][:, 0]

    an_x_var = s3_data['cartesian_an']['x_an'][:]
    an_y_var = s3_data['cartesian_an']['y_an'][:]

    spl = RectBivariateSpline(tx_y_var, tx_x_var[::-1], szn[:, ::-1].filled(0))
    interpolated = spl.ev(an_y_var.compressed(),
                          an_x_var.compressed())
    interpolated = np.ma.masked_invalid(interpolated, copy=False)
    szn = np.ma.empty(an_y_var.shape,
                      dtype=szn.dtype)
    szn[np.logical_not(np.ma.getmaskarray(an_y_var))] = interpolated
    szn.mask = an_y_var.mask
    return szn

def make_night_mask(s3_data):
    solar_zenith_angle = interpolate_szn(s3_data)
    return solar_zenith_angle, solar_zenith_angle.filled(0) >= proc_const.day_night_angle

def interpolate_vza(s3_data):
    sat_zn = s3_data['geometry_tn']['sat_zenith_tn'][:]

    tx_x_var = s3_data['cartesian_tx']['x_tx'][0, :]
    tx_y_var = s3_data['cartesian_tx']['y_tx'][:, 0]

    an_x_var = s3_data['cartesian_an']['x_an'][:]
    an_y_var = s3_data['cartesian_an']['y_an'][:]

    spl = RectBivariateSpline(tx_y_var, tx_x_var[::-1], sat_zn[:, ::-1].filled(0))
    interpolated = spl.ev(an_y_var.compressed(),
                          an_x_var.compressed())
    interpolated = np.ma.masked_invalid(interpolated, copy=False)
    sat_zn = np.ma.empty(an_y_var.shape,
                      dtype=sat_zn.dtype)
    sat_zn[np.logical_not(np.ma.getmaskarray(an_y_var))] = interpolated
    sat_zn.mask = an_y_var.mask
    return sat_zn

def make_vza_mask(s3_data):
    view_zenith_angles = interpolate_vza(s3_data)
    return view_zenith_angles, view_zenith_angles.filled(100) <= 22

def detect_hotspots(s3_data):
    # fill nan's with zero.  Solar constant comes from SLSTR viscal product
    thresh = proc_const.swir_thresh_sls / (100 * np.pi) * 254.23103333  # convert ref threhsold to rad
    return s3_data['S5_radiance_an']['S5_radiance_an'][:].filled(0) > thresh

def detect_hotspots_adaptive(ds, sza_mask, vza_mask):

    # first get unillimunated central swath data
    valid_mask = ds != -999
    useable_data = ds[sza_mask & vza_mask & valid_mask]

    # get threshold
    thresh = np.mean(useable_data) + 4 * np.std(useable_data)
    logger.info('Threshold: ' + str(thresh))

    # get all data above threshold
    above_thresh = ds > thresh

    # find flares
    return sza_mask & vza_mask & valid_mask & above_thresh

def detect_hotspots_non_parametric(ds, sza_mask, vza_mask):

    # first get unillimunated central swath data
    valid_mask = ds != -999
    useable_data = ds[sza_mask & vza_mask & valid_mask]

    # find smallest interval between records for scene
    unique_values = np.unique(useable_data)
    unique_values.sort()
    diff = unique_values[1:] - unique_values[0:-1]
    smallest_diff = np.min(diff)

    logger.info('smallest diff scene: ' + str(smallest_diff))

    # find threshold for data
    useable_data.sort()
    top_subset = useable_data[-1000:]
    bottom_subset = useable_data[:1000]

    diff_top = top_subset[1:] - top_subset[0:-1]
    diff_bottom = bottom_subset[1:] - bottom_subset[0:-1]

    # check the max differences for both top and bottom 1k radiances
    # if they are similar - within 5 DNs, then assume no flares.  This
    # is reasonable as flares increase as step functions (with changes likely
    # to be greater than a couple of DN points).  This helps remove scenes
    # with no flaring activity in them.  Otherwise, get hundreds of invalid
    # datapoints
    max_diff_top = np.max(diff_top)
    max_diff_bottom = np.max(diff_bottom)

    logger.info('Max diff top 1k: ' + str())
    logger.info('Max diff bottom 1k: ' + str())

    if max_diff_top <= 5*max_diff_bottom:
        return None

    diff_mask = diff_top > smallest_diff
    thresh = np.min(top_subset[1:][diff_mask])

    logger.info('Threshold using scene smallest diff: ' + str(thresh))

    # get hotspots
    above_thresh = ds > thresh

    return sza_mask & vza_mask & valid_mask & above_thresh

def flare_data(s3_data, sza, vza, hotspot_mask):

    lines, samples = np.where(hotspot_mask)
    lats = s3_data['geodetic_an']['latitude_an'][:][hotspot_mask]
    lons = s3_data['geodetic_an']['longitude_an'][:][hotspot_mask]
    sza = sza[hotspot_mask]
    vza = vza[hotspot_mask]
    S5 = s3_data['S5_radiance_an']['S5_radiance_an'][:][hotspot_mask]
    S6 = s3_data['S6_radiance_an']['S6_radiance_an'][:][hotspot_mask]

    df = pd.DataFrame()
    datasets = [lines, samples, lats, lons, sza, vza, S5, S6]
    names = ['lines', 'samples', 'lats', 'lons', 'sza', 'vza', 'S5', 'S6']
    for k,v in zip(names, datasets):
        df[k] = v

    return df

def main():

    path_to_data = sys.argv[1]
    path_to_output = sys.argv[2]
    path_to_temp = sys.argv[3]

    output_fname = path_to_data.split('/')[-1].split('.')[0] + '_hotspots.csv'
    csv_path = os.path.join(path_to_output, output_fname)

    s3_data = extract_zip(path_to_data, path_to_temp)

    # load in S5 and S6 channels
    s5_data = s3_data['S5_radiance_an']['S5_radiance_an'][:].filled(-999)

    # get vza and sza masks
    sza, sza_mask = make_night_mask(s3_data)
    if sza_mask.max() == 0:  # all daytime data
        with open(csv_path, "w"):
            pass
        return
    vza, vza_mask = make_vza_mask(s3_data)

    # get the hotspot data for both channels and then generate the mask
    try:
        hotspot_mask = detect_hotspots_non_parametric(s5_data, sza_mask, vza_mask)

        # if no valid hotspots then return none
        if hotspot_mask is None:
            with open(csv_path, "w"):
                pass
            return
        logger.info('N flares detected: ' + str(np.sum(hotspot_mask)))
    except:
        # will fail if no hotspots but still record the processing of the file
        logger.info('N flares detected: 0')
        with open(csv_path, "w"):
            pass
        return

    df = flare_data(s3_data, sza, vza, hotspot_mask)

    logger.info(output_fname)
    df.to_csv(csv_path, index=False)

if __name__ == "__main__":
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)
    logger = logging.getLogger(__name__)
    main()
<commit_msg>slstr scenes now only assumed to contain flares if the max radiance difference between two pixels for the brightest 1000 pixels is at least 5 time greater than the max radiance difference between the bottom 1000 pixels<commit_after>#!/home/users/dnfisher/soft/virtual_envs/ggf/bin/python2

'''
Detects hotspot locations in a given file.  Used later
to detect flares based on hotspot persistency.
'''

import sys
import os
import logging
import zipfile
import shutil
import glob

import numpy as np
from netCDF4 import Dataset
import pandas as pd
from scipy.interpolate import RectBivariateSpline

import src.config.constants as proc_const

def extract_zip(input_zip, path_to_temp):
    data_dict = {}
    to_extract = ["S5_radiance_an.nc", "S6_radiance_an.nc",
                  "geodetic_an.nc", "geometry_tn.nc",
                  "cartesian_an.nc", "cartesian_tx.nc",
                  "indices_an.nc", "flags_an.nc"]
    with zipfile.ZipFile(input_zip) as input:
        for name in input.namelist():
            split_name = name.split('/')[-1]
            if split_name in to_extract:
                var_name = split_name.split('.')[0]
                source = Dataset(input.extract(name, path_to_temp))
                data_dict[var_name] = source

    # remove the unzip files
    dir_to_remove = os.path.join(path_to_temp, input_zip.split('/')[-1].replace('zip', 'SEN3'))
    if os.path.isdir(dir_to_remove):  # test if the path points to a directory
        shutil.rmtree(dir_to_remove, ignore_errors=True)
    else:  # normal file
        os.remove(dir_to_remove)

    return data_dict

def interpolate_szn(s3_data):
    szn = s3_data['geometry_tn']['solar_zenith_tn'][:]

    tx_x_var = s3_data['cartesian_tx']['x_tx'][0, :]
    tx_y_var = s3_data['cartesian_tx']['y_tx'][:, 0]

    an_x_var = s3_data['cartesian_an']['x_an'][:]
    an_y_var = s3_data['cartesian_an']['y_an'][:]

    spl = RectBivariateSpline(tx_y_var, tx_x_var[::-1], szn[:, ::-1].filled(0))
    interpolated = spl.ev(an_y_var.compressed(),
                          an_x_var.compressed())
    interpolated = np.ma.masked_invalid(interpolated, copy=False)
    szn = np.ma.empty(an_y_var.shape,
                      dtype=szn.dtype)
    szn[np.logical_not(np.ma.getmaskarray(an_y_var))] = interpolated
    szn.mask = an_y_var.mask
    return szn

def make_night_mask(s3_data):
    solar_zenith_angle = interpolate_szn(s3_data)
    return solar_zenith_angle, solar_zenith_angle.filled(0) >= proc_const.day_night_angle

def interpolate_vza(s3_data):
    sat_zn = s3_data['geometry_tn']['sat_zenith_tn'][:]

    tx_x_var = s3_data['cartesian_tx']['x_tx'][0, :]
    tx_y_var = s3_data['cartesian_tx']['y_tx'][:, 0]

    an_x_var = s3_data['cartesian_an']['x_an'][:]
    an_y_var = s3_data['cartesian_an']['y_an'][:]

    spl = RectBivariateSpline(tx_y_var, tx_x_var[::-1], sat_zn[:, ::-1].filled(0))
    interpolated = spl.ev(an_y_var.compressed(),
                          an_x_var.compressed())
    interpolated = np.ma.masked_invalid(interpolated, copy=False)
    sat_zn = np.ma.empty(an_y_var.shape,
                      dtype=sat_zn.dtype)
    sat_zn[np.logical_not(np.ma.getmaskarray(an_y_var))] = interpolated
    sat_zn.mask = an_y_var.mask
    return sat_zn

def make_vza_mask(s3_data):
    view_zenith_angles = interpolate_vza(s3_data)
    return view_zenith_angles, view_zenith_angles.filled(100) <= 22

def detect_hotspots(s3_data):
    # fill nan's with zero.  Solar constant comes from SLSTR viscal product
    thresh = proc_const.swir_thresh_sls / (100 * np.pi) * 254.23103333  # convert ref threhsold to rad
    return s3_data['S5_radiance_an']['S5_radiance_an'][:].filled(0) > thresh

def detect_hotspots_adaptive(ds, sza_mask, vza_mask):

    # first get unillimunated central swath data
    valid_mask = ds != -999
    useable_data = ds[sza_mask & vza_mask & valid_mask]

    # get threshold
    thresh = np.mean(useable_data) + 4 * np.std(useable_data)
    logger.info('Threshold: ' + str(thresh))

    # get all data above threshold
    above_thresh = ds > thresh

    # find flares
    return sza_mask & vza_mask & valid_mask & above_thresh

def detect_hotspots_non_parametric(ds, sza_mask, vza_mask):

    # first get unillimunated central swath data
    valid_mask = ds != -999
    useable_data = ds[sza_mask & vza_mask & valid_mask]

    # find smallest interval between records for scene
    unique_values = np.unique(useable_data)
    unique_values.sort()
    diff = unique_values[1:] - unique_values[0:-1]
    smallest_diff = np.min(diff)

    logger.info('smallest diff scene: ' + str(smallest_diff))

    # find threshold for data
    useable_data.sort()
    top_subset = useable_data[-1000:]
    bottom_subset = useable_data[:1000]

    diff_top = top_subset[1:] - top_subset[0:-1]
    diff_bottom = bottom_subset[1:] - bottom_subset[0:-1]

    # check the max differences for both top and bottom 1k radiances
    # if they are similar - within 5 DNs, then assume no flares.  This
    # is reasonable as flares increase as step functions (with changes likely
    # to be greater than a couple of DN points).  This helps remove scenes
    # with no flaring activity in them.  Otherwise, get hundreds of invalid
    # datapoints
    max_diff_top = np.max(diff_top)
    max_diff_bottom = np.max(diff_bottom)

    logger.info('Max diff top 1k: ' + str())
    logger.info('Max diff bottom 1k: ' + str())

    if max_diff_top <= 5*max_diff_bottom:
        return None

    diff_mask = diff_top > smallest_diff
    thresh = np.min(top_subset[1:][diff_mask])

    logger.info('Threshold using scene smallest diff: ' + str(thresh))

    # get hotspots
    above_thresh = ds > thresh

    return sza_mask & vza_mask & valid_mask & above_thresh

def flare_data(s3_data, sza, vza, hotspot_mask):

    lines, samples = np.where(hotspot_mask)
    lats = s3_data['geodetic_an']['latitude_an'][:][hotspot_mask]
    lons = s3_data['geodetic_an']['longitude_an'][:][hotspot_mask]
    sza = sza[hotspot_mask]
    vza = vza[hotspot_mask]
    S5 = s3_data['S5_radiance_an']['S5_radiance_an'][:][hotspot_mask]
    S6 = s3_data['S6_radiance_an']['S6_radiance_an'][:][hotspot_mask]

    df = pd.DataFrame()
    datasets = [lines, samples, lats, lons, sza, vza, S5, S6]
    names = ['lines', 'samples', 'lats', 'lons', 'sza', 'vza', 'S5', 'S6']
    for k,v in zip(names, datasets):
        df[k] = v

    return df

def main():

    path_to_data = sys.argv[1]
    path_to_output = sys.argv[2]
    path_to_temp = sys.argv[3]

    output_fname = path_to_data.split('/')[-1].split('.')[0] + '_hotspots.csv'
    csv_path = os.path.join(path_to_output, output_fname)

    s3_data = extract_zip(path_to_data, path_to_temp)

    # load in S5 and S6 channels
    s5_data = s3_data['S5_radiance_an']['S5_radiance_an'][:].filled(-999)

    # get vza and sza masks
    sza, sza_mask = make_night_mask(s3_data)
    if sza_mask.max() == 0:  # all daytime data
        with open(csv_path, "w"):
            pass
        return
    vza, vza_mask = make_vza_mask(s3_data)

    # get the hotspot data for both channels and then generate the mask
    try:
        hotspot_mask = detect_hotspots_non_parametric(s5_data, sza_mask, vza_mask)

        # if no valid hotspots then return none
        if hotspot_mask is None:
            logger.info('N flares detected: 0 - not enough difference between max and min pixel values')
            with open(csv_path, "w"):
                pass
            return
        logger.info('N flares detected: ' + str(np.sum(hotspot_mask)))
    except:
        # will fail if no hotspots but still record the processing of the file
        logger.info('N flares detected: 0')
        with open(csv_path, "w"):
            pass
        return

    df = flare_data(s3_data, sza, vza, hotspot_mask)

    logger.info(output_fname)
    df.to_csv(csv_path, index=False)

if __name__ == "__main__":
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)
    logger = logging.getLogger(__name__)
    main()
<|endoftext|>


<commit_before>from ingestors.base import Ingestor
from ingestors.support.temp import TempFileSupport
from ingestors.support.shell import ShellSupport
from ingestors.support.ole import OLESupport
from ingestors.directory import DirectoryIngestor

class OutlookPSTIngestor(Ingestor, TempFileSupport, ShellSupport, OLESupport):
    MIME_TYPES = [
        'application/vnd.ms-outlook'
    ]
    EXTENSIONS = ['pst', 'ost', 'pab']
    BASE_SCORE = 5
    COMMAND_TIMEOUT = 12 * 60 * 60

    def ingest(self, file_path):
        self.extract_ole_metadata(file_path)
        self.result.flag(self.result.FLAG_DIRECTORY)
        with self.create_temp_dir() as temp_dir:
            if self.result.mime_type is None:
                self.result.mime_type = 'application/vnd.ms-outlook'
            self.exec_command('readpst',
                              '-M',  # make subfolders, files per message
                              '-D',  # include deleted
                              '-r',  # recursive structure
                              '-8',  # utf-8 where possible
                              '-b',
                              '-q',  # quiet
                              '-o', temp_dir,
                              file_path)
            self.manager.delegate(DirectoryIngestor, self.result, temp_dir)
<commit_msg>Make readpst partial output ingest.<commit_after>from ingestors.base import Ingestor
from ingestors.support.temp import TempFileSupport
from ingestors.support.shell import ShellSupport
from ingestors.support.ole import OLESupport
from ingestors.directory import DirectoryIngestor

class OutlookPSTIngestor(Ingestor, TempFileSupport, ShellSupport, OLESupport):
    MIME_TYPES = [
        'application/vnd.ms-outlook'
    ]
    EXTENSIONS = ['pst', 'ost', 'pab']
    BASE_SCORE = 5
    COMMAND_TIMEOUT = 12 * 60 * 60

    def ingest(self, file_path):
        self.extract_ole_metadata(file_path)
        self.result.flag(self.result.FLAG_DIRECTORY)
        with self.create_temp_dir() as temp_dir:
            if self.result.mime_type is None:
                self.result.mime_type = 'application/vnd.ms-outlook'
            try:
                self.exec_command('readpst',
                                  '-e',  # make subfolders, files per message
                                  '-D',  # include deleted
                                  '-r',  # recursive structure
                                  '-8',  # utf-8 where possible
                                  '-b',
                                  '-q',  # quiet
                                  '-o', temp_dir,
                                  file_path)
                self.manager.delegate(DirectoryIngestor, self.result, temp_dir)
            except Exception:
                # Handle partially extracted archives.
                self.manager.delegate(DirectoryIngestor, self.result, temp_dir)
                raise
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2022 The init2winit Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Hyperparameter management logic."""

import json

from absl import logging
from init2winit.dataset_lib import datasets
from init2winit.init_lib import initializers
from init2winit.model_lib import models
from ml_collections.config_dict import config_dict
from tensorflow.io import gfile

def build_hparams(model_name,
                  initializer_name,
                  dataset_name,
                  hparam_file,
                  hparam_overrides,
                  input_pipeline_hps=None):
  """Build experiment hyperparameters.

  Args:
    model_name: the string model name.
    initializer_name: the string initializer name.
    dataset_name: the string dataset name.
    hparam_file: the string to the hyperparameter override file (possibly on
      CNS).
    hparam_overrides: a dict of hyperparameter override names/values, or a JSON
      string encoding of this hyperparameter override dict. Note that this is
      applied after the hyperparameter file overrides.
    input_pipeline_hps: a dict of hyperparameters for performance tuning.

  Returns:
    A ConfigDict of experiment hyperparameters.
  """
  model_hps = models.get_model_hparams(model_name)
  initializer_hps = initializers.get_initializer_hparams(initializer_name)
  dataset_hps = datasets.get_dataset_hparams(dataset_name)
  input_pipeline_hps = input_pipeline_hps or config_dict.ConfigDict()

  merged_dict = {}

  hps_dicts = [
      hps.to_dict()
      for hps in [model_hps, initializer_hps, dataset_hps, input_pipeline_hps]
  ]

  total_hps = 0
  for hps_dict in hps_dicts:
    merged_dict.update(hps_dict)
    total_hps += len(hps_dict.keys())

  # Check that all provided have no overlap.
  if total_hps != len(merged_dict.keys()):
    raise ValueError('There is overlap in the provided hparams.')

  # Convert to the Shallue and Lee label smoothing style.
  if merged_dict.get('use_shallue_label_smoothing', False):
    num_classes = merged_dict['output_shape'][-1]
    merged_dict['label_smoothing'] *= num_classes / float(num_classes - 1)

  merged = config_dict.ConfigDict(merged_dict)
  merged.lock()

  # Subconfig "opt_hparams" and "lr_hparams" are allowed to add new fields.
  for key in ['opt_hparams', 'lr_hparams']:
    if key not in merged:
      with merged.unlocked():
        merged[key] = config_dict.ConfigDict()

  for key in ['opt_hparams', 'lr_hparams']:
    merged[key].unlock()

  if hparam_file:
    logging.info('Loading hparams from %s', hparam_file)
    with gfile.GFile(hparam_file, 'r') as f:
      hparam_dict = json.load(f)
      merged.update_from_flattened_dict(hparam_dict)

  if hparam_overrides:
    if isinstance(hparam_overrides, str):
      hparam_overrides = json.loads(hparam_overrides)

    # If the user is changing the learning rate schedule or optimizer. We must
    # wipe all of the keys from the old dictionary.
    if 'lr_hparams.schedule' in hparam_overrides and merged[
        'lr_hparams']['schedule'] != hparam_overrides[
            'lr_hparams.schedule']:
      merged['lr_hparams'] = {}
    if 'optimizer' in hparam_overrides and merged[
        'optimizer'] != hparam_overrides['optimizer']:
      merged['opt_hparams'] = {}
    merged.update_from_flattened_dict(hparam_overrides)

  return merged
<commit_msg>Fix `lr_hparams.schedule` check in `hyperparameters.py` in case the key `schedule` is not in `merged_hps.lr_hparams`.<commit_after># coding=utf-8
# Copyright 2022 The init2winit Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Hyperparameter management logic."""

import json

from absl import logging
from init2winit.dataset_lib import datasets
from init2winit.init_lib import initializers
from init2winit.model_lib import models
from ml_collections.config_dict import config_dict
from tensorflow.io import gfile

def build_hparams(model_name,
                  initializer_name,
                  dataset_name,
                  hparam_file,
                  hparam_overrides,
                  input_pipeline_hps=None):
  """Build experiment hyperparameters.

  Args:
    model_name: the string model name.
    initializer_name: the string initializer name.
    dataset_name: the string dataset name.
    hparam_file: the string to the hyperparameter override file (possibly on
      CNS).
    hparam_overrides: a dict of hyperparameter override names/values, or a JSON
      string encoding of this hyperparameter override dict. Note that this is
      applied after the hyperparameter file overrides.
    input_pipeline_hps: a dict of hyperparameters for performance tuning.

  Returns:
    A ConfigDict of experiment hyperparameters.
  """
  model_hps = models.get_model_hparams(model_name)
  initializer_hps = initializers.get_initializer_hparams(initializer_name)
  dataset_hps = datasets.get_dataset_hparams(dataset_name)
  input_pipeline_hps = input_pipeline_hps or config_dict.ConfigDict()

  merged_dict = {}

  hps_dicts = [
      hps.to_dict()
      for hps in [model_hps, initializer_hps, dataset_hps, input_pipeline_hps]
  ]

  total_hps = 0
  for hps_dict in hps_dicts:
    merged_dict.update(hps_dict)
    total_hps += len(hps_dict.keys())

  # Check that all provided have no overlap.
  if total_hps != len(merged_dict.keys()):
    raise ValueError('There is overlap in the provided hparams.')

  # Convert to the Shallue and Lee label smoothing style.
  if merged_dict.get('use_shallue_label_smoothing', False):
    num_classes = merged_dict['output_shape'][-1]
    merged_dict['label_smoothing'] *= num_classes / float(num_classes - 1)

  merged = config_dict.ConfigDict(merged_dict)
  merged.lock()

  # Subconfig "opt_hparams" and "lr_hparams" are allowed to add new fields.
  for key in ['opt_hparams', 'lr_hparams']:
    if key not in merged:
      with merged.unlocked():
        merged[key] = config_dict.ConfigDict()

  for key in ['opt_hparams', 'lr_hparams']:
    merged[key].unlock()

  if hparam_file:
    logging.info('Loading hparams from %s', hparam_file)
    with gfile.GFile(hparam_file, 'r') as f:
      hparam_dict = json.load(f)
      merged.update_from_flattened_dict(hparam_dict)

  if hparam_overrides:
    if isinstance(hparam_overrides, str):
      hparam_overrides = json.loads(hparam_overrides)

    # If the user is changing the learning rate schedule or optimizer. We must
    # wipe all of the keys from the old dictionary.
    if 'lr_hparams.schedule' in hparam_overrides and merged[
        'lr_hparams'].get('schedule') != hparam_overrides[
            'lr_hparams.schedule']:
      merged['lr_hparams'] = {}
    if 'optimizer' in hparam_overrides and merged[
        'optimizer'] != hparam_overrides['optimizer']:
      merged['opt_hparams'] = {}
    merged.update_from_flattened_dict(hparam_overrides)

  return merged
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# pylint: disable=W0703,C0412

"""Linter configurations."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

import fnmatch
from multiprocessing.pool import ThreadPool as Pool
import os.path
import subprocess
import sys
import time
import traceback

# Use the built-in version of scandir/walk if possible, otherwise
# use the scandir module version
try:
    from os import scandir, walk
except ImportError:
    from scandir import scandir, walk # noqa

from inlineplz import parsers
from inlineplz import message

HERE = os.path.dirname(__file__)

PATTERNS = {
    'python': ['*.py'],
    'javascript': ['*.js'],
    'json': ['*.json'],
    'yaml': ['*.yaml', '*.yml'],
    'rst': ['*.rst'],
    'markdown': ['*.md'],
    'stylus': ['*.styl'],
    'robotframework': ['*.robot'],
    'gherkin': ['*.feature'],
    'docker': ['*Dockerfile'],
    'shell': ['*.sh', '*.zsh', '*.ksh', '*.bsh', '*.csh', '*.bash']
}

LINTERS = {
    'prospector': {
        'install': [
            ['pip', 'install', 'prospector[with_everything]'],
            ['pip', 'install', 'prospector']
        ],
        'help': ['prospector', '-h'],
        'run': ['prospector', '--zero-exit', '-o', 'json'],
        'rundefault': ['prospector', '--zero-exit', '-o', 'json', '-P',
                       '{config_dir}/.prospector.yaml'],
        'dotfiles': ['.prospector.yaml'],
        'parser': parsers.ProspectorParser,
        'language': 'python',
        'autorun': True,
        'run_per_file': False
    },
    'eslint': {
        'install': [['npm', 'install'], ['npm', 'install', 'eslint']],
        'help': [os.path.normpath('./node_modules/.bin/eslint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/eslint'), '.', '-f', 'json'],
        'rundefault': [os.path.normpath('./node_modules/.bin/eslint'), '.', '-f', 'json', '-c',
                       '{config_dir}/.eslintrc.js'],
        'dotfiles': [
            '.eslintrc.yml',
            '.eslintrc.yaml',
            '.eslintignore',
            '.eslintrc',
            '.eslintrc.js',
            '.eslintrc.json'
        ],
        'parser': parsers.ESLintParser,
        'language': 'javascript',
        'autorun': True,
        'run_per_file': False
    },
    'gherkin-lint': {
        'install': [['npm', 'install'], ['npm', 'install', 'gherkin-lint']],
        'help': [os.path.normpath('./node_modules/.bin/gherkin-lint'), '--help'],
        'run': [os.path.normpath('./node_modules/.bin/gherkin-lint'), '.', '-f', 'json'],
        'rundefault': [os.path.normpath('./node_modules/.bin/gherkin-lint'), '.', '-f', 'json', '-c',
                       '{config_dir}/.gherkin-lintrc'],
        'dotfiles': [
            '.gherkin-lintrc'
        ],
        'parser': parsers.GherkinLintParser,
        'language': 'gherkin',
        'autorun': True,
        'run_per_file': False
    },
    'jshint': {
        'install': [['npm', 'install'], ['npm', 'install', 'jshint']],
        'help': [os.path.normpath('./node_modules/.bin/jshint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/jshint'), '.', '--reporter', 'checkstyle'],
        'rundefault': [os.path.normpath('./node_modules/.bin/jshint'), '.', '--reporter', 'checkstyle', '-c',
                       '{config_dir}/.jshintrc'],
        'dotfiles': ['.jshintrc'],
        'parser': parsers.JSHintParser,
        'language': 'javascript',
        'autorun': False,
        'run_per_file': False
    },
    'jscs': {
        'install': [['npm', 'install'], ['npm', 'install', 'jscs']],
        'help': [os.path.normpath('./node_modules/.bin/jscs'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/jscs'), '.', '-r', 'json', '-m', '-1', '-v'],
        'rundefault': [
            os.path.normpath('./node_modules/.bin/jscs'),
            '.', '-r', 'json', '-m', '-1', '-v', '-c',
            '{config_dir}/.jscsrc'
        ],
        'dotfiles': ['.jscsrc', '.jscs.json'],
        'parser': parsers.JSCSParser,
        'language': 'javascript',
        'autorun': False,
        'run_per_file': False
    },
    'jsonlint': {
        'install': [['npm', 'install'], ['npm', 'install', 'jsonlint']],
        'help': [os.path.normpath('./node_modules/.bin/jsonlint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/jsonlint'), '-c', '-q'],
        'rundefault': [os.path.normpath('./node_modules/.bin/jsonlint'), '-c', '-q'],
        'dotfiles': [],
        'parser': parsers.JSONLintParser,
        'language': 'json',
        'autorun': True,
        'run_per_file': True
    },
    'yaml-lint': {
        'install': [['gem', 'install', 'yaml-lint']],
        'help': ['yaml-lint'],
        'run': ['yaml-lint', '-q'],
        'rundefault': ['yaml-lint', '-q'],
        'dotfiles': [],
        'parser': parsers.YAMLLintParser,
        'language': 'yaml',
        'autorun': True,
        'run_per_file': True
    },
    'rst-lint': {
        'install': [['pip', 'install', 'restructuredtext_lint']],
        'help': ['rst-lint', '-h'],
        'run': ['rst-lint', '--format', 'json'],
        'rundefault': ['rst-lint', '--format', 'json'],
        'dotfiles': [],
        'parser': parsers.RSTLintParser,
        'language': 'rst',
        'autorun': True,
        'run_per_file': True
    },
    'markdownlint': {
        'install': [['gem', 'install', 'mdl']],
        'help': ['mdl', '-h'],
        'run': ['mdl', '.'],
        'rundefault': ['mdl', '.', '-c', '{config_dir}/.mdlrc'],
        'dotfiles': ['.mdlrc'],
        'parser': parsers.MarkdownLintParser,
        'language': 'markdown',
        'autorun': True,
        'run_per_file': False
    },
    'stylint': {
        'install': [['npm', 'install'], ['npm', 'install', 'stylint']],
        'help': [os.path.normpath('./node_modules/.bin/stylint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/stylint')],
        'rundefault': [
            os.path.normpath('./node_modules/.bin/stylint'),
            '-c',
            '{config_dir}/.stylintrc'
        ],
        'dotfiles': ['.stylintrc'],
        'parser': parsers.StylintParser,
        'language': 'stylus',
        'autorun': True,
        'run_per_file': False
    },
    'rflint': {
        'install': [['pip', 'install', 'robotframework-lint']],
        'help': ['rflint', '--help'],
        'run': ['rflint'],
        'rundefault': ['rflint', '-A', '{config_dir}/.rflint'],
        'dotfiles': ['.rflint'],
        'parser': parsers.RobotFrameworkLintParser,
        'language': 'robotframework',
        'autorun': True,
        'run_per_file': True
    },
    'dockerfile_lint': {
        'install': [['npm', 'install'], ['npm', 'install', 'dockerfile_lint']],
        'help': [os.path.normpath('./node_modules/.bin/dockerfile_lint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/dockerfile_lint'), '-j', '-f'],
        'rundefault': [os.path.normpath('./node_modules/.bin/dockerfile_lint'), '-j', '-f'],
        'dotfiles': [],
        'parser': parsers.DockerfileLintParser,
        'language': 'docker',
        'autorun': True,
        'run_per_file': True
    },
    'shellcheck': {
        'install': [
            ['cabal', 'update'],
            ['cabal', 'install', 'shellcheck'],
            ['apt-get', 'install', 'shellcheck'],
            ['dnf', 'install', 'shellcheck'],
            ['brew', 'install', 'shellcheck'],
            ['port', 'install', 'shellcheck'],
            ['zypper', 'in', 'ShellCheck'],
        ],
        'help': ['shellcheck', '-V'],
        'run': ['shellcheck', '-f', 'json'],
        'rundefault': ['shellcheck', '-f', 'json'],
        'dotfiles': [],
        'parser': parsers.ShellcheckParser,
        'language': 'shell',
        'autorun': True,
        'run_per_file': True
    },
    'bandit': {
        'install': [['pip', 'install', 'bandit']],
        'help': ['bandit', '-h'],
        'run': ['bandit', '-f', 'json', '-iii', '-ll', '-r', '.'],
        'rundefault': ['bandit', '-f', 'json', '-iii', '-ll', '-r', '.', '-c',
                       '{config_dir}/bandit.yaml'],
        'dotfiles': ['bandit.yaml'],
        'parser': parsers.BanditParser,
        'language': 'python',
        'autorun': True,
        'run_per_file': False
    }
}

def run_command(command, log_on_fail=False, log_all=False):
    shell = False
    if os.name == 'nt':
        shell = True
    proc = subprocess.Popen(
        command,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        shell=shell,
        env=os.environ
    )
    stdout, stderr = proc.communicate()
    stdout = stdout.decode('utf-8', errors='replace')
    stderr = stderr.decode('utf-8', errors='replace')
    output = '{}\n{}'.format(stdout, stderr).strip()
    if (log_on_fail and proc.returncode) or log_all:
        print(output.encode('ascii', errors='replace'))
        sys.stdout.flush()
    return proc.returncode, output

def performance_hacks():
    # https://github.com/npm/npm/issues/11283
    # npm's progress bar makes npm installs much slower
    try:
        run_command(['npm', 'set', 'progress=false'])
    except Exception:
        pass

def should_ignore_path(path, ignore_paths):
    for ignore_path in ignore_paths:
        if (
            os.path.relpath(path).startswith(ignore_path) or
            path.startswith(ignore_path) or
            fnmatch.fnmatch(path, ignore_path)
        ):
            return True
    return False

def run_per_file(config, ignore_paths=None, path=None, config_dir=None):
    ignore_paths = ignore_paths or []
    path = path or os.getcwd()
    cmd = run_config(config, config_dir)
    print(cmd)
    run_cmds = []
    patterns = PATTERNS.get(config.get('language'))
    paths = all_filenames_in_dir(path=path, ignore_paths=ignore_paths)
    for pattern in patterns:
        for filepath in fnmatch.filter(paths, pattern):
            run_cmds.append(cmd + [filepath])
    pool = Pool()

    def result(run_cmd):
        _, out = run_command(run_cmd)
        return run_cmd[-1], out

    output = pool.map(result, run_cmds)
    return output

def linters_to_run(install=False, autorun=False, ignore_paths=None):
    linters = set()
    if not autorun:
        for linter, config in LINTERS.items():
            if (installed(config) or install) and dotfiles_exist(config):
                linters.add(linter)
    else:
        dotfilefound = {}
        for linter, config in LINTERS.items():
            if dotfiles_exist(config):
                dotfilefound[config.get('language')] = True
                linters.add(linter)
        filenames = all_filenames_in_dir(path=os.getcwd(), ignore_paths=ignore_paths)
        for linter, config in LINTERS.items():
            if not dotfilefound.get(config.get('language')) and should_autorun(config, filenames):
                linters.add(linter)
    return linters

def all_filenames_in_dir(path=None, ignore_paths=None):
    path = path or os.getcwd()
    # http://stackoverflow.com/a/2186565
    paths = set()
    for root, dirnames, filenames in os.walk(path, topdown=True):
        try:
            for ignore in ignore_paths:
                dirnames.remove(ignore)
        except ValueError:
            pass
        if should_ignore_path(root, ignore_paths):
            continue
        for filename in filenames:
            paths.add(os.path.join(root, filename))
    return paths

def should_autorun(config, filenames):
    patterns = PATTERNS.get(config.get('language'))
    if config.get('autorun'):
        for pattern in patterns:
            if fnmatch.filter(filenames, pattern):
                return True
    return False

def dotfiles_exist(config, path=None):
    path = path or os.getcwd()
    return any(dotfile.strip() in os.listdir(path) for dotfile in config.get('dotfiles'))

PREVIOUS_INSTALL_COMMANDS = []

def install_linter(config):
    for install_cmd in config.get('install'):
        if install_cmd in PREVIOUS_INSTALL_COMMANDS:
            continue
        PREVIOUS_INSTALL_COMMANDS.append(install_cmd)
        if not installed(config):
            try:
                run_command(install_cmd, log_all=True)
            except OSError:
                print('Install failed: {0}\n{1}'.format(install_cmd, traceback.format_exc()))
        else:
            return

def installed(config):
    try:
        returncode, _ = run_command(config.get('help'))
        return returncode == 0
    except (subprocess.CalledProcessError, OSError):
        return False

def run_config(config, config_dir):
    if dotfiles_exist(config) and config.get('run'):
        return config.get('run')
    if not (config_dir and dotfiles_exist(config, config_dir)):
        config_dir = os.path.abspath(os.path.join(HERE, 'config'))
    return [
        os.path.normpath(item.format(config_dir=config_dir))
        for item in (config.get('rundefault') or config.get('run'))
    ]

def lint(install=False, autorun=False, ignore_paths=None, config_dir=None):
    messages = message.Messages()
    performance_hacks()
    for linter in linters_to_run(install, autorun, ignore_paths):
        print('Running linter: {0}'.format(linter))
        sys.stdout.flush()
        start = time.time()
        output = ''
        config = LINTERS.get(linter)
        try:
            if (install or autorun) and config.get('install'):
                install_linter(config)
            if config.get('run_per_file'):
                output = run_per_file(config, ignore_paths, config_dir)
            else:
                cmd = run_config(config, config_dir)
                print(cmd)
                _, output = run_command(cmd)
        except Exception:
            traceback.print_exc()
            print(str(output).encode('ascii', errors='replace'))
        print('Installation and running of {0} took {1} seconds'.format(linter, int(time.time() - start)))
        sys.stdout.flush()
        start = time.time()
        try:
            if output:
                linter_messages = config.get('parser')().parse(output)
                # prepend linter name to message content
                linter_messages = {
                    (msg[0], msg[1], '{0}: {1}'.format(linter, msg[2])) for msg in linter_messages
                }
                messages.add_messages(linter_messages)
        except Exception:
            traceback.print_exc()
            print(str(output).encode('ascii', errors='replace'))
        print('Parsing of {0} took {1} seconds'.format(linter, int(time.time() - start)))
    return messages.get_messages()
<commit_msg>run 'bundle install' during ruby gem installs (#147)<commit_after># -*- coding: utf-8 -*-
# pylint: disable=W0703,C0412

"""Linter configurations."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

import fnmatch
from multiprocessing.pool import ThreadPool as Pool
import os.path
import subprocess
import sys
import time
import traceback

# Use the built-in version of scandir/walk if possible, otherwise
# use the scandir module version
try:
    from os import scandir, walk
except ImportError:
    from scandir import scandir, walk # noqa

from inlineplz import parsers
from inlineplz import message

HERE = os.path.dirname(__file__)

PATTERNS = {
    'python': ['*.py'],
    'javascript': ['*.js'],
    'json': ['*.json'],
    'yaml': ['*.yaml', '*.yml'],
    'rst': ['*.rst'],
    'markdown': ['*.md'],
    'stylus': ['*.styl'],
    'robotframework': ['*.robot'],
    'gherkin': ['*.feature'],
    'docker': ['*Dockerfile'],
    'shell': ['*.sh', '*.zsh', '*.ksh', '*.bsh', '*.csh', '*.bash']
}

LINTERS = {
    'prospector': {
        'install': [
            ['pip', 'install', 'prospector[with_everything]'],
            ['pip', 'install', 'prospector']
        ],
        'help': ['prospector', '-h'],
        'run': ['prospector', '--zero-exit', '-o', 'json'],
        'rundefault': ['prospector', '--zero-exit', '-o', 'json', '-P',
                       '{config_dir}/.prospector.yaml'],
        'dotfiles': ['.prospector.yaml'],
        'parser': parsers.ProspectorParser,
        'language': 'python',
        'autorun': True,
        'run_per_file': False
    },
    'eslint': {
        'install': [['npm', 'install'], ['npm', 'install', 'eslint']],
        'help': [os.path.normpath('./node_modules/.bin/eslint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/eslint'), '.', '-f', 'json'],
        'rundefault': [os.path.normpath('./node_modules/.bin/eslint'), '.', '-f', 'json', '-c',
                       '{config_dir}/.eslintrc.js'],
        'dotfiles': [
            '.eslintrc.yml',
            '.eslintrc.yaml',
            '.eslintignore',
            '.eslintrc',
            '.eslintrc.js',
            '.eslintrc.json'
        ],
        'parser': parsers.ESLintParser,
        'language': 'javascript',
        'autorun': True,
        'run_per_file': False
    },
    'gherkin-lint': {
        'install': [['npm', 'install'], ['npm', 'install', 'gherkin-lint']],
        'help': [os.path.normpath('./node_modules/.bin/gherkin-lint'), '--help'],
        'run': [os.path.normpath('./node_modules/.bin/gherkin-lint'), '.', '-f', 'json'],
        'rundefault': [os.path.normpath('./node_modules/.bin/gherkin-lint'), '.', '-f', 'json', '-c',
                       '{config_dir}/.gherkin-lintrc'],
        'dotfiles': [
            '.gherkin-lintrc'
        ],
        'parser': parsers.GherkinLintParser,
        'language': 'gherkin',
        'autorun': True,
        'run_per_file': False
    },
    'jshint': {
        'install': [['npm', 'install'], ['npm', 'install', 'jshint']],
        'help': [os.path.normpath('./node_modules/.bin/jshint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/jshint'), '.', '--reporter', 'checkstyle'],
        'rundefault': [os.path.normpath('./node_modules/.bin/jshint'), '.', '--reporter', 'checkstyle', '-c',
                       '{config_dir}/.jshintrc'],
        'dotfiles': ['.jshintrc'],
        'parser': parsers.JSHintParser,
        'language': 'javascript',
        'autorun': False,
        'run_per_file': False
    },
    'jscs': {
        'install': [['npm', 'install'], ['npm', 'install', 'jscs']],
        'help': [os.path.normpath('./node_modules/.bin/jscs'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/jscs'), '.', '-r', 'json', '-m', '-1', '-v'],
        'rundefault': [
            os.path.normpath('./node_modules/.bin/jscs'),
            '.', '-r', 'json', '-m', '-1', '-v', '-c',
            '{config_dir}/.jscsrc'
        ],
        'dotfiles': ['.jscsrc', '.jscs.json'],
        'parser': parsers.JSCSParser,
        'language': 'javascript',
        'autorun': False,
        'run_per_file': False
    },
    'jsonlint': {
        'install': [['npm', 'install'], ['npm', 'install', 'jsonlint']],
        'help': [os.path.normpath('./node_modules/.bin/jsonlint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/jsonlint'), '-c', '-q'],
        'rundefault': [os.path.normpath('./node_modules/.bin/jsonlint'), '-c', '-q'],
        'dotfiles': [],
        'parser': parsers.JSONLintParser,
        'language': 'json',
        'autorun': True,
        'run_per_file': True
    },
    'yaml-lint': {
        'install': [['bundle', 'install'], ['gem', 'install', 'yaml-lint']],
        'help': ['yaml-lint'],
        'run': ['yaml-lint', '-q'],
        'rundefault': ['yaml-lint', '-q'],
        'dotfiles': [],
        'parser': parsers.YAMLLintParser,
        'language': 'yaml',
        'autorun': True,
        'run_per_file': True
    },
    'rst-lint': {
        'install': [['pip', 'install', 'restructuredtext_lint']],
        'help': ['rst-lint', '-h'],
        'run': ['rst-lint', '--format', 'json'],
        'rundefault': ['rst-lint', '--format', 'json'],
        'dotfiles': [],
        'parser': parsers.RSTLintParser,
        'language': 'rst',
        'autorun': True,
        'run_per_file': True
    },
    'markdownlint': {
        'install': [['bundle', 'install'], ['gem', 'install', 'mdl']],
        'help': ['mdl', '-h'],
        'run': ['mdl', '.'],
        'rundefault': ['mdl', '.', '-c', '{config_dir}/.mdlrc'],
        'dotfiles': ['.mdlrc'],
        'parser': parsers.MarkdownLintParser,
        'language': 'markdown',
        'autorun': True,
        'run_per_file': False
    },
    'stylint': {
        'install': [['npm', 'install'], ['npm', 'install', 'stylint']],
        'help': [os.path.normpath('./node_modules/.bin/stylint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/stylint')],
        'rundefault': [
            os.path.normpath('./node_modules/.bin/stylint'),
            '-c',
            '{config_dir}/.stylintrc'
        ],
        'dotfiles': ['.stylintrc'],
        'parser': parsers.StylintParser,
        'language': 'stylus',
        'autorun': True,
        'run_per_file': False
    },
    'rflint': {
        'install': [['pip', 'install', 'robotframework-lint']],
        'help': ['rflint', '--help'],
        'run': ['rflint'],
        'rundefault': ['rflint', '-A', '{config_dir}/.rflint'],
        'dotfiles': ['.rflint'],
        'parser': parsers.RobotFrameworkLintParser,
        'language': 'robotframework',
        'autorun': True,
        'run_per_file': True
    },
    'dockerfile_lint': {
        'install': [['npm', 'install'], ['npm', 'install', 'dockerfile_lint']],
        'help': [os.path.normpath('./node_modules/.bin/dockerfile_lint'), '-h'],
        'run': [os.path.normpath('./node_modules/.bin/dockerfile_lint'), '-j', '-f'],
        'rundefault': [os.path.normpath('./node_modules/.bin/dockerfile_lint'), '-j', '-f'],
        'dotfiles': [],
        'parser': parsers.DockerfileLintParser,
        'language': 'docker',
        'autorun': True,
        'run_per_file': True
    },
    'shellcheck': {
        'install': [
            ['cabal', 'update'],
            ['cabal', 'install', 'shellcheck'],
            ['apt-get', 'install', 'shellcheck'],
            ['dnf', 'install', 'shellcheck'],
            ['brew', 'install', 'shellcheck'],
            ['port', 'install', 'shellcheck'],
            ['zypper', 'in', 'ShellCheck'],
        ],
        'help': ['shellcheck', '-V'],
        'run': ['shellcheck', '-f', 'json'],
        'rundefault': ['shellcheck', '-f', 'json'],
        'dotfiles': [],
        'parser': parsers.ShellcheckParser,
        'language': 'shell',
        'autorun': True,
        'run_per_file': True
    },
    'bandit': {
        'install': [['pip', 'install', 'bandit']],
        'help': ['bandit', '-h'],
        'run': ['bandit', '-f', 'json', '-iii', '-ll', '-r', '.'],
        'rundefault': ['bandit', '-f', 'json', '-iii', '-ll', '-r', '.', '-c',
                       '{config_dir}/bandit.yaml'],
        'dotfiles': ['bandit.yaml'],
        'parser': parsers.BanditParser,
        'language': 'python',
        'autorun': True,
        'run_per_file': False
    }
}

def run_command(command, log_on_fail=False, log_all=False):
    shell = False
    if os.name == 'nt':
        shell = True
    proc = subprocess.Popen(
        command,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        shell=shell,
        env=os.environ
    )
    stdout, stderr = proc.communicate()
    stdout = stdout.decode('utf-8', errors='replace')
    stderr = stderr.decode('utf-8', errors='replace')
    output = '{}\n{}'.format(stdout, stderr).strip()
    if (log_on_fail and proc.returncode) or log_all:
        print(output.encode('ascii', errors='replace'))
        sys.stdout.flush()
    return proc.returncode, output

def performance_hacks():
    # https://github.com/npm/npm/issues/11283
    # npm's progress bar makes npm installs much slower
    try:
        run_command(['npm', 'set', 'progress=false'])
    except Exception:
        pass

def should_ignore_path(path, ignore_paths):
    for ignore_path in ignore_paths:
        if (
            os.path.relpath(path).startswith(ignore_path) or
            path.startswith(ignore_path) or
            fnmatch.fnmatch(path, ignore_path)
        ):
            return True
    return False

def run_per_file(config, ignore_paths=None, path=None, config_dir=None):
    ignore_paths = ignore_paths or []
    path = path or os.getcwd()
    cmd = run_config(config, config_dir)
    print(cmd)
    run_cmds = []
    patterns = PATTERNS.get(config.get('language'))
    paths = all_filenames_in_dir(path=path, ignore_paths=ignore_paths)
    for pattern in patterns:
        for filepath in fnmatch.filter(paths, pattern):
            run_cmds.append(cmd + [filepath])
    pool = Pool()

    def result(run_cmd):
        _, out = run_command(run_cmd)
        return run_cmd[-1], out

    output = pool.map(result, run_cmds)
    return output

def linters_to_run(install=False, autorun=False, ignore_paths=None):
    linters = set()
    if not autorun:
        for linter, config in LINTERS.items():
            if (installed(config) or install) and dotfiles_exist(config):
                linters.add(linter)
    else:
        dotfilefound = {}
        for linter, config in LINTERS.items():
            if dotfiles_exist(config):
                dotfilefound[config.get('language')] = True
                linters.add(linter)
        filenames = all_filenames_in_dir(path=os.getcwd(), ignore_paths=ignore_paths)
        for linter, config in LINTERS.items():
            if not dotfilefound.get(config.get('language')) and should_autorun(config, filenames):
                linters.add(linter)
    return linters

def all_filenames_in_dir(path=None, ignore_paths=None):
    path = path or os.getcwd()
    # http://stackoverflow.com/a/2186565
    paths = set()
    for root, dirnames, filenames in os.walk(path, topdown=True):
        try:
            for ignore in ignore_paths:
                dirnames.remove(ignore)
        except ValueError:
            pass
        if should_ignore_path(root, ignore_paths):
            continue
        for filename in filenames:
            paths.add(os.path.join(root, filename))
    return paths

def should_autorun(config, filenames):
    patterns = PATTERNS.get(config.get('language'))
    if config.get('autorun'):
        for pattern in patterns:
            if fnmatch.filter(filenames, pattern):
                return True
    return False

def dotfiles_exist(config, path=None):
    path = path or os.getcwd()
    return any(dotfile.strip() in os.listdir(path) for dotfile in config.get('dotfiles'))

PREVIOUS_INSTALL_COMMANDS = []

def install_linter(config):
    for install_cmd in config.get('install'):
        if install_cmd in PREVIOUS_INSTALL_COMMANDS:
            continue
        PREVIOUS_INSTALL_COMMANDS.append(install_cmd)
        if not installed(config):
            try:
                run_command(install_cmd, log_all=True)
            except OSError:
                print('Install failed: {0}\n{1}'.format(install_cmd, traceback.format_exc()))
        else:
            return

def installed(config):
    try:
        returncode, _ = run_command(config.get('help'))
        return returncode == 0
    except (subprocess.CalledProcessError, OSError):
        return False

def run_config(config, config_dir):
    if dotfiles_exist(config) and config.get('run'):
        return config.get('run')
    if not (config_dir and dotfiles_exist(config, config_dir)):
        config_dir = os.path.abspath(os.path.join(HERE, 'config'))
    return [
        os.path.normpath(item.format(config_dir=config_dir))
        for item in (config.get('rundefault') or config.get('run'))
    ]

def lint(install=False, autorun=False, ignore_paths=None, config_dir=None):
    messages = message.Messages()
    performance_hacks()
    for linter in linters_to_run(install, autorun, ignore_paths):
        print('Running linter: {0}'.format(linter))
        sys.stdout.flush()
        start = time.time()
        output = ''
        config = LINTERS.get(linter)
        try:
            if (install or autorun) and config.get('install'):
                install_linter(config)
            if config.get('run_per_file'):
                output = run_per_file(config, ignore_paths, config_dir)
            else:
                cmd = run_config(config, config_dir)
                print(cmd)
                _, output = run_command(cmd)
        except Exception:
            traceback.print_exc()
            print(str(output).encode('ascii', errors='replace'))
        print('Installation and running of {0} took {1} seconds'.format(linter, int(time.time() - start)))
        sys.stdout.flush()
        start = time.time()
        try:
            if output:
                linter_messages = config.get('parser')().parse(output)
                # prepend linter name to message content
                linter_messages = {
                    (msg[0], msg[1], '{0}: {1}'.format(linter, msg[2])) for msg in linter_messages
                }
                messages.add_messages(linter_messages)
        except Exception:
            traceback.print_exc()
            print(str(output).encode('ascii', errors='replace'))
        print('Parsing of {0} took {1} seconds'.format(linter, int(time.time() - start)))
    return messages.get_messages()
<|endoftext|>


<commit_before>import pymongo, bson
QUERY = {'deleted': False}
FIELDS = [
    'document_id',
    'docket_id',
    'agency',
    'title',
    'details.fr_publish_date',
    'type',
    'views.entities',
    'attachments.views.entities',
    'submitter_entities'
]

# class to work around the mincemeat requirement that the data be structured as a dictionary
class MongoSource(object):
    def __init__(self, db):
        self.cache = {}
        self.db = db

    def __iter__(self):
        _cache = self.cache

        def gen():
            for doc in self.db.docs.find(QUERY, FIELDS):
                doc_id = str(doc['_id'])
                _cache[doc_id] = doc
                yield doc_id

        return gen()

    def __getitem__(self, key):
        out = self.cache[key]
        del self.cache[key]
        return out

def mapfn(key, document):
    import isoweek
    from collections import defaultdict
    import itertools

    # pre-computed date data
    doc_type = document.get('type', None)
    doc_date = document.get('details', {}).get('fr_publish_date', None)
    doc_week = isoweek.Week(*(doc_date.isocalendar()[:-1])) if doc_date else None
    doc_week_range = (doc_week.monday().isoformat(), doc_week.sunday().isoformat()) if doc_week else None
    doc_month = doc_date.isoformat()[:7] if doc_date else None

    ### COLLECTION: dockets ###
    docket_info = {
        'count': 1,
        'type_breakdown': {str(doc_type): 1},
        'rules': [{
            'date': doc_date.date().isoformat() if doc_date else None,
            'type': doc_type,
            'id': document['document_id'],
            'title': document['title']
        }] if doc_type in ['rule', 'proposed_rule'] else [],
        'weeks': [(doc_week_range, 1)],
        'date_range': [doc_date, doc_date],
        'text_entities': {},
        'submitter_entities': {}
    }

    ### COLLECTION: agencies ###
    agency_info = {
        'count': 1,
        'date_range': [doc_date, doc_date],
        'months': [(doc_month, 1)],
        'type_breakdown': {str(doc_type): 1},
        'text_entities': {},
        'submitter_entities': {}
    }

    # entity data for dockets and agencies
    # text entities
    views = itertools.chain.from_iterable([document.get('views', [])] + [attachment.get('views', []) for attachment in document.get('attachments', [])])
    for view in views:
        for entity in view.get('entities', []):
            docket_info['text_entities'][entity] = 1
            agency_info['text_entities'][entity] = 1

    # submitters
    for entity in document.get('submitter_entities', []):
        docket_info['submitter_entities'][entity] = 1
        agency_info['submitter_entities'][entity] = 1

    yield ('dockets', document['docket_id']), docket_info
    yield ('agencies', document.get('agency', None)), agency_info

    ### COLLECTION: entities ###
    entities = set(docket_info['text_entities'].keys())
    entities.update(docket_info['submitter_entities'].keys())

    for entity in entities:
        text_count = docket_info['text_entities'].get(entity, 0)
        submitter_count = docket_info['submitter_entities'].get(entity, 0)

        entity_info = {
            'text_mentions': {
                'count': text_count,
                'agencies': {document.get('agency', None): text_count},
                'dockets': {document['docket_id']: text_count},
                'months': [(doc_month, text_count)],
                'date_range': [doc_date, doc_date] if text_count else [None, None]
            },
            'submitter_mentions': {
                'count': submitter_count,
                'agencies': {document.get('agency', None): submitter_count},
                'dockets': {document['docket_id']: submitter_count},
                'months': [(doc_month, submitter_count)],
                'date_range': [doc_date, doc_date] if submitter_count else [None, None]
            }
        }
        yield ('entities', entity), entity_info

def reducefn(key, documents):
    from collections import defaultdict
    import datetime

    def min_date(*args):
        a = [arg for arg in args if arg is not None]
        if not a:
            return None
        else:
            return min(a)

    def max_date(*args):
        a = [arg for arg in args if arg is not None]
        if not a:
            return None
        else:
            return max(a)

    ### COLLECTION: dockets ###
    if key[0] == 'dockets':
        out = {
            'count': 0,
            'type_breakdown': defaultdict(int),
            'rules': [],
            'weeks': defaultdict(int),
            'date_range': [None, None],
            'text_entities': defaultdict(int),
            'submitter_entities': defaultdict(int)
        }
        if documents:
            out['date_range'] = documents[0]['date_range']

        for value in documents:
            out['count'] += value['count']

            for doc_type, count in value['type_breakdown'].iteritems():
                out['type_breakdown'][doc_type] += count

            out['rules'].extend(value['rules'])

            for week, count in dict(value['weeks']).iteritems():
                out['weeks'][week] += count

            for entity, count in value['text_entities'].iteritems():
                out['text_entities'][entity] += count

            for entity, count in value['submitter_entities'].iteritems():
                out['submitter_entities'][entity] += count

            out['date_range'][0] = min_date(out['date_range'][0], value['date_range'][0])
            out['date_range'][1] = max_date(out['date_range'][1], value['date_range'][1])

        out['rules'] = sorted(out['rules'], key=lambda x: x['date'])

        out['weeks'] = sorted(out['weeks'].items(), key=lambda x: x[0][0] if x[0] else datetime.date.min.isoformat())
        return out

    ### COLLECTION: agencies ###
    if key[0] == 'agencies':
        out = {
            'count': 0,
            'type_breakdown': defaultdict(int),
            'months': defaultdict(int),
            'date_range': [None, None],
            'text_entities': defaultdict(int),
            'submitter_entities': defaultdict(int)
        }
        if documents:
            out['date_range'] = documents[0]['date_range']

        for value in documents:
            out['count'] += value['count']

            for doc_type, count in value['type_breakdown'].iteritems():
                out['type_breakdown'][doc_type] += count

            for month, count in dict(value['months']).iteritems():
                out['months'][month] += count

            for entity, count in value['text_entities'].iteritems():
                out['text_entities'][entity] += count

            for entity, count in value['submitter_entities'].iteritems():
                out['submitter_entities'][entity] += count

            out['date_range'][0] = min_date(out['date_range'][0], value['date_range'][0])
            out['date_range'][1] = max_date(out['date_range'][1], value['date_range'][1])

        out['months'] = sorted(out['months'].items(), key=lambda x: x[0] if x[0] else datetime.date.min.isoformat())
        return out

    ### COLLECTION: entities ###
    elif key[0] == 'entities':
        out = {
            'text_mentions': {
                'count': 0,
                'agencies': defaultdict(int),
                'dockets': defaultdict(int),
                'months': defaultdict(int),
                'date_range': [None, None]
            },
            'submitter_mentions': {
                'count': 0,
                'agencies': defaultdict(int),
                'dockets': defaultdict(int),
                'months': defaultdict(int),
                'date_range': [None, None]
            }
        }
        for value in documents:
            for mention_type in ['text_mentions', 'submitter_mentions']:
                out[mention_type]['count'] += value[mention_type]['count']
                for agency, count in value[mention_type]['agencies'].iteritems():
                    if value[mention_type]['agencies'][agency]:
                        out[mention_type]['agencies'][agency] += value[mention_type]['agencies'][agency]
                for docket, count in value[mention_type]['dockets'].iteritems():
                    if value[mention_type]['dockets'][docket]:
                        out[mention_type]['dockets'][docket] += value[mention_type]['dockets'][docket]
                months_dict = dict(value[mention_type]['months'])
                for month, count in months_dict.iteritems():
                    if months_dict[month]:
                        out[mention_type]['months'][month] += months_dict[month]
                out[mention_type]['date_range'][0] = min_date(out[mention_type]['date_range'][0], value[mention_type]['date_range'][0])
                out[mention_type]['date_range'][1] = max_date(out[mention_type]['date_range'][1], value[mention_type]['date_range'][1])

        for mention_type in ['text_mentions', 'submitter_mentions']:
            out[mention_type]['months'] = sorted(out[mention_type]['months'].items(), key=lambda x: x[0] if x[0] else datetime.date.min.isoformat())

        return out

if __name__ == '__main__':
    db = pymongo.Connection().regulations

    import mincemeat
    s = mincemeat.SqliteServer('/tmp/test.db')
    s.mapfn = mapfn
    s.reducefn = reducefn
    s.datasource = MongoSource(db)

    results = s.run_server()

    for key, value in results:
        collection = key[0]
        _id = key[1]
        if not _id:
            continue

        try:
            db[collection].update(
                {
                    '_id': _id
                },
                {
                    '$set': {'stats': value}
                },
                upsert=True,
                safe=True
            )
        except bson.errors.InvalidDocument:
            print value
            raise<commit_msg>In map reduce, rename 'rules' to 'fr_docs' and include notices in it.<commit_after>import pymongo, bson
QUERY = {'deleted': False}
FIELDS = [
    'document_id',
    'docket_id',
    'agency',
    'title',
    'details.fr_publish_date',
    'type',
    'views.entities',
    'attachments.views.entities',
    'submitter_entities'
]

# class to work around the mincemeat requirement that the data be structured as a dictionary
class MongoSource(object):
    def __init__(self, db):
        self.cache = {}
        self.db = db

    def __iter__(self):
        _cache = self.cache

        def gen():
            for doc in self.db.docs.find(QUERY, FIELDS):
                doc_id = str(doc['_id'])
                _cache[doc_id] = doc
                yield doc_id

        return gen()

    def __getitem__(self, key):
        out = self.cache[key]
        del self.cache[key]
        return out

def mapfn(key, document):
    import isoweek
    from collections import defaultdict
    import itertools

    # pre-computed date data
    doc_type = document.get('type', None)
    doc_date = document.get('details', {}).get('fr_publish_date', None)
    doc_week = isoweek.Week(*(doc_date.isocalendar()[:-1])) if doc_date else None
    doc_week_range = (doc_week.monday().isoformat(), doc_week.sunday().isoformat()) if doc_week else None
    doc_month = doc_date.isoformat()[:7] if doc_date else None

    ### COLLECTION: dockets ###
    docket_info = {
        'count': 1,
        'type_breakdown': {str(doc_type): 1},
        'fr_docs': [{
            'date': doc_date.date().isoformat() if doc_date else None,
            'type': doc_type,
            'id': document['document_id'],
            'title': document['title']
        }] if doc_type in ['notice', 'rule', 'proposed_rule'] else [],
        'weeks': [(doc_week_range, 1)],
        'date_range': [doc_date, doc_date],
        'text_entities': {},
        'submitter_entities': {}
    }

    ### COLLECTION: agencies ###
    agency_info = {
        'count': 1,
        'date_range': [doc_date, doc_date],
        'months': [(doc_month, 1)],
        'type_breakdown': {str(doc_type): 1},
        'text_entities': {},
        'submitter_entities': {}
    }

    # entity data for dockets and agencies
    # text entities
    views = itertools.chain.from_iterable([document.get('views', [])] + [attachment.get('views', []) for attachment in document.get('attachments', [])])
    for view in views:
        for entity in view.get('entities', []):
            docket_info['text_entities'][entity] = 1
            agency_info['text_entities'][entity] = 1

    # submitters
    for entity in document.get('submitter_entities', []):
        docket_info['submitter_entities'][entity] = 1
        agency_info['submitter_entities'][entity] = 1

    yield ('dockets', document['docket_id']), docket_info
    yield ('agencies', document.get('agency', None)), agency_info

    ### COLLECTION: entities ###
    entities = set(docket_info['text_entities'].keys())
    entities.update(docket_info['submitter_entities'].keys())

    for entity in entities:
        text_count = docket_info['text_entities'].get(entity, 0)
        submitter_count = docket_info['submitter_entities'].get(entity, 0)

        entity_info = {
            'text_mentions': {
                'count': text_count,
                'agencies': {document.get('agency', None): text_count},
                'dockets': {document['docket_id']: text_count},
                'months': [(doc_month, text_count)],
                'date_range': [doc_date, doc_date] if text_count else [None, None]
            },
            'submitter_mentions': {
                'count': submitter_count,
                'agencies': {document.get('agency', None): submitter_count},
                'dockets': {document['docket_id']: submitter_count},
                'months': [(doc_month, submitter_count)],
                'date_range': [doc_date, doc_date] if submitter_count else [None, None]
            }
        }
        yield ('entities', entity), entity_info

def reducefn(key, documents):
    from collections import defaultdict
    import datetime

    def min_date(*args):
        a = [arg for arg in args if arg is not None]
        if not a:
            return None
        else:
            return min(a)

    def max_date(*args):
        a = [arg for arg in args if arg is not None]
        if not a:
            return None
        else:
            return max(a)

    ### COLLECTION: dockets ###
    if key[0] == 'dockets':
        out = {
            'count': 0,
            'type_breakdown': defaultdict(int),
            'fr_docs': [],
            'weeks': defaultdict(int),
            'date_range': [None, None],
            'text_entities': defaultdict(int),
            'submitter_entities': defaultdict(int)
        }
        if documents:
            out['date_range'] = documents[0]['date_range']

        for value in documents:
            out['count'] += value['count']

            for doc_type, count in value['type_breakdown'].iteritems():
                out['type_breakdown'][doc_type] += count

            out['fr_docs'].extend(value['fr_docs'])

            for week, count in dict(value['weeks']).iteritems():
                out['weeks'][week] += count

            for entity, count in value['text_entities'].iteritems():
                out['text_entities'][entity] += count

            for entity, count in value['submitter_entities'].iteritems():
                out['submitter_entities'][entity] += count

            out['date_range'][0] = min_date(out['date_range'][0], value['date_range'][0])
            out['date_range'][1] = max_date(out['date_range'][1], value['date_range'][1])

        out['fr_docs'] = sorted(out['fr_docs'], key=lambda x: x['date'])

        out['weeks'] = sorted(out['weeks'].items(), key=lambda x: x[0][0] if x[0] else datetime.date.min.isoformat())
        return out

    ### COLLECTION: agencies ###
    if key[0] == 'agencies':
        out = {
            'count': 0,
            'type_breakdown': defaultdict(int),
            'months': defaultdict(int),
            'date_range': [None, None],
            'text_entities': defaultdict(int),
            'submitter_entities': defaultdict(int)
        }
        if documents:
            out['date_range'] = documents[0]['date_range']

        for value in documents:
            out['count'] += value['count']

            for doc_type, count in value['type_breakdown'].iteritems():
                out['type_breakdown'][doc_type] += count

            for month, count in dict(value['months']).iteritems():
                out['months'][month] += count

            for entity, count in value['text_entities'].iteritems():
                out['text_entities'][entity] += count

            for entity, count in value['submitter_entities'].iteritems():
                out['submitter_entities'][entity] += count

            out['date_range'][0] = min_date(out['date_range'][0], value['date_range'][0])
            out['date_range'][1] = max_date(out['date_range'][1], value['date_range'][1])

        out['months'] = sorted(out['months'].items(), key=lambda x: x[0] if x[0] else datetime.date.min.isoformat())
        return out

    ### COLLECTION: entities ###
    elif key[0] == 'entities':
        out = {
            'text_mentions': {
                'count': 0,
                'agencies': defaultdict(int),
                'dockets': defaultdict(int),
                'months': defaultdict(int),
                'date_range': [None, None]
            },
            'submitter_mentions': {
                'count': 0,
                'agencies': defaultdict(int),
                'dockets': defaultdict(int),
                'months': defaultdict(int),
                'date_range': [None, None]
            }
        }
        for value in documents:
            for mention_type in ['text_mentions', 'submitter_mentions']:
                out[mention_type]['count'] += value[mention_type]['count']
                for agency, count in value[mention_type]['agencies'].iteritems():
                    if value[mention_type]['agencies'][agency]:
                        out[mention_type]['agencies'][agency] += value[mention_type]['agencies'][agency]
                for docket, count in value[mention_type]['dockets'].iteritems():
                    if value[mention_type]['dockets'][docket]:
                        out[mention_type]['dockets'][docket] += value[mention_type]['dockets'][docket]
                months_dict = dict(value[mention_type]['months'])
                for month, count in months_dict.iteritems():
                    if months_dict[month]:
                        out[mention_type]['months'][month] += months_dict[month]
                out[mention_type]['date_range'][0] = min_date(out[mention_type]['date_range'][0], value[mention_type]['date_range'][0])
                out[mention_type]['date_range'][1] = max_date(out[mention_type]['date_range'][1], value[mention_type]['date_range'][1])

        for mention_type in ['text_mentions', 'submitter_mentions']:
            out[mention_type]['months'] = sorted(out[mention_type]['months'].items(), key=lambda x: x[0] if x[0] else datetime.date.min.isoformat())

        return out

if __name__ == '__main__':
    db = pymongo.Connection().regulations

    import mincemeat
    s = mincemeat.SqliteServer('/tmp/test.db')
    s.mapfn = mapfn
    s.reducefn = reducefn
    s.datasource = MongoSource(db)

    results = s.run_server()

    for key, value in results:
        collection = key[0]
        _id = key[1]
        if not _id:
            continue

        try:
            db[collection].update(
                {
                    '_id': _id
                },
                {
                    '$set': {'stats': value}
                },
                upsert=True,
                safe=True
            )
        except bson.errors.InvalidDocument:
            print value
            raise<|endoftext|>


<commit_before>"""
Creates DBPedia labels-types file of the following format:

{ LABEL: [Type1, Type2, ...], ...}

For example:

Tramore:             Town, Settlement, PopulatedPlace, Place
Tramore,_Ireland:    Town, Settlement, PopulatedPlace, Place
"""
import codecs

import subprocess
import urllib
from collections import defaultdict

TYPE_FILES = ['instance-types_en.nt.bz2', 'instance-types-transitive_en.nt.bz2']
EXCLUDES = {'Agent', 'TimePeriod', 'PersonFunction', 'Year'}

typed_entities = defaultdict(lambda: {'types': set(), 'redirects': []})
# BZ2File module cannot process multi-stream files, so use subprocess
for types_file in TYPE_FILES:
    p = subprocess.Popen('bzcat -q ' + types_file, shell=True, stdout=subprocess.PIPE)
    for line in p.stdout:
        if '<BAD URI: Illegal character' in line:
            continue
        try:
            uri, predicate, type_uri = line.split(' ', 2)
        except:
            continue
        if '<http://schema.org/Person>' in type_uri:
            type_uri = '<http://dbpedia.org/ontology/Person>   '
        if 'http://dbpedia.org/ontology/' not in type_uri:
            continue
        if 'http://dbpedia.org/ontology/Wikidata' in type_uri:
            continue
        if 'http://dbpedia.org/ontology/Location' in type_uri:
            continue
        uri = urllib.unquote(uri.replace('<http://dbpedia.org/resource/', '')[:-1])
        type_uri = type_uri.replace('<http://dbpedia.org/ontology/', '')[:-4]
        if '(disambiguation)' in uri:
            continue
        if type_uri in EXCLUDES:
            continue

        uri = uri.decode('utf-8')
        typed_entities[uri]['types'].add(type_uri)

REDIRECTS_FILE = 'redirects_transitive_en.nt.bz2'
# BZ2File module cannot process multi-stream files, so use subprocess
p = subprocess.Popen('bzcat -q ' + REDIRECTS_FILE, shell=True, stdout=subprocess.PIPE)
for line in p.stdout:
    try:
        uri_redirect, predicate, uri_canon = line.split(' ', 2)
    except:
        continue
    name_redirect = urllib.unquote(uri_redirect.replace('<http://dbpedia.org/resource/', '')[:-1])
    name_canon = urllib.unquote(uri_canon.replace('<http://dbpedia.org/resource/', '')[:-4])
    if '(disambiguation)' in name_redirect:
        continue

    typed_entities[name_canon.decode('utf-8')]['redirects'].append(name_redirect.decode('utf-8'))

dbpedia_data = codecs.open('dbpedia_data.txt', 'w', 'utf-8')

for uri, value_dict in typed_entities.iteritems():
    dbpedia_data.write(uri+'\t'+' '.join(value_dict['types'])+'\t'+' '.join(value_dict['redirects'])+'\n')

dbpedia_data.close()
<commit_msg>remove ambiguous things<commit_after>"""
Creates DBPedia labels-types file of the following format:

{ LABEL: [Type1, Type2, ...], ...}

For example:

Tramore:             Town, Settlement, PopulatedPlace, Place
Tramore,_Ireland:    Town, Settlement, PopulatedPlace, Place
"""
import codecs

import subprocess
import urllib
from collections import defaultdict

TYPE_FILES = ['instance-types_en.nt.bz2', 'instance-types-transitive_en.nt.bz2']
EXCLUDES = {'Agent', 'TimePeriod', 'PersonFunction', 'Year'}

typed_entities = defaultdict(lambda: {'types': set(), 'redirects': []})
# BZ2File module cannot process multi-stream files, so use subprocess
for types_file in TYPE_FILES:
    p = subprocess.Popen('bzcat -q ' + types_file, shell=True, stdout=subprocess.PIPE)
    for line in p.stdout:
        if '<BAD URI: Illegal character' in line:
            continue
        try:
            uri, predicate, type_uri = line.split(' ', 2)
        except:
            continue
        if '<http://schema.org/Person>' in type_uri:
            type_uri = '<http://dbpedia.org/ontology/Person>   '
        if 'http://dbpedia.org/ontology/' not in type_uri:
            continue
        if 'http://dbpedia.org/ontology/Wikidata' in type_uri:
            continue
        if 'http://dbpedia.org/ontology/Location' in type_uri:
            continue
        uri = urllib.unquote(uri.replace('<http://dbpedia.org/resource/', '')[:-1])
        type_uri = type_uri.replace('<http://dbpedia.org/ontology/', '')[:-4]
        if '(disambiguation)' in uri:
            continue
        if type_uri in EXCLUDES:
            continue

        uri = uri.decode('utf-8')
        typed_entities[uri]['types'].add(type_uri)

REDIRECTS_FILE = 'redirects_transitive_en.nt.bz2'
# BZ2File module cannot process multi-stream files, so use subprocess
p = subprocess.Popen('bzcat -q ' + REDIRECTS_FILE, shell=True, stdout=subprocess.PIPE)
for line in p.stdout:
    try:
        uri_redirect, predicate, uri_canon = line.split(' ', 2)
    except:
        continue
    name_redirect = urllib.unquote(uri_redirect.replace('<http://dbpedia.org/resource/', '')[:-1])
    name_canon = urllib.unquote(uri_canon.replace('<http://dbpedia.org/resource/', '')[:-4])
    if '(disambiguation)' in name_redirect:
        # delete other references if exist
        del typed_entities[name_canon.decode('utf-8')]
        continue

    typed_entities[name_canon.decode('utf-8')]['redirects'].append(name_redirect.decode('utf-8'))

dbpedia_data = codecs.open('dbpedia_data.txt', 'w', 'utf-8')

for uri, value_dict in typed_entities.iteritems():
    dbpedia_data.write(uri+'\t'+' '.join(value_dict['types'])+'\t'+' '.join(value_dict['redirects'])+'\n')

dbpedia_data.close()
<|endoftext|>