<commit_before>from __future__ import print_function
from __future__ import unicode_literals

import random
import re

class CrayCrayWriter(object):
    """An object that can generate stories!"""

    def __init__(self):
        """Init all the data we need"""

        self.characters = ["Kate","Andrew","Micah", "Elizabeth"]
        self.actions = ["Punch","Kick","Yell","Ignor","Us", "Observ", "Star", "Patt", "Cuddl", "Kiss", "Talk", "Fart", "Laugh", "Hat", "Lov"]
        self.actionsDict = { "Punch": "the",
                         "Kick": "the",
                         "Yell": "at the",
                         "Ignor": "the",
                         "Us": "the",
                         "Observ": "the",
                         "Star": "at the",
                         "Patt": "the",
                         "Cuddl": "the",
                         "Kiss": "the",
                         "Talk": "to the",
                         "Fart": "at the",
                         "Laugh":"at the",
                         "Hat":"the",
                         "Lov":"the"
                       }
        self.objects = ["Couch","Chair","TV","Computer","Cat","Dog"]
        self.rooms = ["Bedroom","Library","Bathroom","Kitchen","Lounge","Playroom","Backyard"]
        self.map = dict()

    def choose_character(self):
        """Randomly return one of the available characters"""

        return self.characters[random.randrange(0, len(self.characters))]

    def choose_action(self):
        """Randomly return one of the possible actions"""

        return self.actions[random.randrange(0, len(self.actions))]

    def choose_object(self):
        """Randomly return one of the possible objects"""

        return self.objects[random.randrange(0, len(self.objects))]

    def choose_room(self):
        """Randomly return one of the possible rooms"""

        return self.rooms[random.randrange(0, len(self.rooms))]

    def decide_map(self):
        """Generate a map of rooms and where they connect"""

        for room in self.rooms:
            self.map[room] = dict()
        for room in self.map:
            self.map[room]["connected"] = list()
        for room in self.map:
            self.map[room]["characters"] = list()
        for room in self.map:
            self.map[room]["objects"] = list()
        for room in self.map:
            iter = len(self.rooms) - 1
            while iter > 0:
                if random.randrange(0, 2) > 0:
                    randRoom = self.choose_room()
                    self.map[room]["connected"].append(randRoom)
                iter = iter - 1
        for room in self.map:
            self.map[room]["connected"] = list(set(self.map[room]["connected"]))
        return self.map

    def decide_room_objects(self):
        """Generate a list of objects and link them to a room"""

        for room in self.map:
            self.map[room]["objects"] = list()
            self.map[room]["objects"].append("Door")
        for room in self.map:
            iter = len(self.rooms) - 1
            while iter > 0:
                if random.randrange(0, 2) > 0:
                    randObj = self.choose_object()
                    self.map[room]["objects"].append(randObj)
                iter = iter - 1
        for room in self.map:
            self.map[room]["objects"] = list(set(self.map[room]["objects"]))
        return self.map

    def check_room_objects(self, room):
        """Check what objects are in the room"""

        return self.map[room]["objects"]

    def check_room_connected(self, room):
        """Check what rooms are connected"""

        return self.map[room]["connected"]

    def remove_character_location(self, character):
        """Remove a characters location from memory"""
        for room in self.map:
            if character in self.map[room]["characters"]:
                  self.map[room]["characters"] = [x for x in self.map[room]["characters"] if x != character]
        return self.map

    def set_character_location(self, room, character):
        """Set a characters location in memory"""

        try:
           self.map[room]["characters"]
        except KeyError:
           for rooms in self.map:
               self.map[rooms]["characters"] = list()
        if type(self.map[room]["characters"]) != list:
            self.map[room]["characters"] = list()
        self.remove_character_location(character)
        self.map[room]["characters"].append(character)
        self.map[room]["characters"] = list(set(self.map[room]["characters"]))
        return self.map

    def check_character_location(self, character):
        """Get a characters location from memory"""
        for room in self.map:
            if character in self.map[room]["characters"]:
                return room
        return "Character Not Found"

    def choose_action(self):
        """Randomly choose an action to do"""

        self.actions.append("move")
        self.actions = list(set(self.actions))
        return self.actions[random.randrange(0, len(self.actions))]

    def choose_object_from_avail(self, room):
        """Randomly choose object from those available in the room"""

        available_objects = self.check_room_objects(room)
        return available_objects[random.randrange(0, len(available_objects))]

    def build_sentence(self):
        """Generate a sentence from available data"""
        character_focus = self.choose_character()
        if self.check_character_location(character_focus) == "Character Not Found":
            character_room = self.choose_room()
            self.set_character_location(character_room, character_focus)
        else:
            character_room = self.check_character_location(character_focus)
        character_action = self.choose_action()
        if character_action == "move":
            connected_rooms = self.check_room_connected(character_room)
            if len(connected_rooms) > 0:
                chosen_room = connected_rooms[random.randrange(0, len(connected_rooms))]
                self.set_character_location(chosen_room, character_focus)
                return " " + character_focus + " moved to the " + chosen_room + "."
            else:
                if random.randrange(0, 10) > 4:
                    chosen_room = self.rooms[random.randrange(0, len(self.rooms))]
                    self.set_character_location(chosen_room, character_focus)
                    return " " + character_focus + " teleported to the " + chosen_room + "."
                else:
                    return " " + character_focus + " cried because stuck."
        character_object = self.choose_object_from_avail(character_room)
        character_avail = self.choose_character_from_avail(character_room, character_focus)
        if character_avail:
            if random.randrange(0, 10) > 4:
                available_chosen = character_avail
                noun = self.actionsDict[character_action].replace("the","")
            else:
                available_chosen = character_object
                noun = self.actionsDict[character_action] + " "
        else:
            available_chosen = character_object
            noun = self.actionsDict[character_action] + " "
        return " " + character_focus + " " + character_action.lower() + "ed " + noun + available_chosen + "."

    def choose_character_from_avail(self, room, character):
        characters_present = self.map[room]["characters"]
        available_list = list()
        for item in characters_present:
            if item != character:
                available_list.append(item)
        if len(available_list) > 0:
            return available_list[random.randrange(0, len(available_list))]
        else:
            return False

    def write_character_locations(self):
        for i in self.characters:
            if self.check_character_location(i) == "Character Not Found":
                character_room = self.choose_room()
                self.set_character_location(character_room, i)
        return [x + " is in the " + self.check_character_location(x) + ".\n" for x in self.characters]

    def build_paragraph(self):
        paragraphIntro = str()
        for item in self.write_character_locations():
            paragraphIntro = paragraphIntro + item
        paragraph = str()
        paragraph_build = list()
        iter = 20
        while iter > 0:
            paragraph_build.append(self.build_sentence())
            iter = iter - 1
        for character in self.characters:
            list_iter = 0
            for item in paragraph_build:
                if character in item[:len(character)+1]:
                    paragraph = paragraph + paragraph_build[list_iter]
                list_iter = list_iter + 1
        return paragraphIntro + "\n" + paragraph[1:]

    def build_book(self):
        bookStream = str()
        while len(bookStream) < (50000 * 8):
            bookStream = bookStream + self.build_paragraph() + "\n\n"
        return bookStream

    def write_book(self):
        with open("outBook.md", "w+") as openFile:
            openFile.write(self.build_book())
        return "outBook.md"

if __name__ == "__main__":
    insane = CrayCrayWriter()
    insane.decide_map()
    insane.decide_room_objects()
    iter = 10
    while iter > 0:
        print(insane.build_paragraph())
        print("\n\n")
        iter = iter - 1
<commit_msg>Added README and LICENSE<commit_after>from __future__ import print_function
from __future__ import unicode_literals

import random
import re

class CrayCrayWriter(object):
    """An object that can generate stories!"""

    def __init__(self):
        """Init all the data we need"""

        self.characters = ["Kate","Andrew","Micah", "Elizabeth"]
        # I know its ugly to have both a list and a dict... I'll get around to fixing it eventually.
        self.actions = ["Punch","Kick","Yell","Ignor","Us", "Observ", "Star", "Patt", "Cuddl", "Kiss", "Talk", "Fart", "Laugh", "Hat", "Lov"]
        self.actionsDict = { "Punch": "the",
                         "Kick": "the",
                         "Yell": "at the",
                         "Ignor": "the",
                         "Us": "the",
                         "Observ": "the",
                         "Star": "at the",
                         "Patt": "the",
                         "Cuddl": "the",
                         "Kiss": "the",
                         "Talk": "to the",
                         "Fart": "at the",
                         "Laugh":"at the",
                         "Hat":"the",
                         "Lov":"the"
                       }
        self.objects = ["Couch","Chair","TV","Computer","Cat","Dog"]
        self.rooms = ["Bedroom","Library","Bathroom","Kitchen","Lounge","Playroom","Backyard"]
        # self.map becomes a list of connections between rooms later on.
        self.map = dict()

    def choose_character(self):
        """Randomly return one of the available characters"""

        return self.characters[random.randrange(0, len(self.characters))]

    def choose_action(self):
        """Randomly return one of the possible actions"""

        return self.actions[random.randrange(0, len(self.actions))]

    def choose_object(self):
        """Randomly return one of the possible objects"""

        return self.objects[random.randrange(0, len(self.objects))]

    def choose_room(self):
        """Randomly return one of the possible rooms"""

        return self.rooms[random.randrange(0, len(self.rooms))]

    def decide_map(self):
        """Generate a map of rooms and where they connect"""

        for room in self.rooms:
            self.map[room] = dict()
        for room in self.map:
            self.map[room]["connected"] = list()
        for room in self.map:
            self.map[room]["characters"] = list()
        for room in self.map:
            self.map[room]["objects"] = list()
        for room in self.map:
            iter = len(self.rooms) - 1
            while iter > 0:
                if random.randrange(0, 2) > 0:
                    randRoom = self.choose_room()
                    self.map[room]["connected"].append(randRoom)
                iter = iter - 1
        for room in self.map:
            self.map[room]["connected"] = list(set(self.map[room]["connected"]))
        return self.map

    def decide_room_objects(self):
        """Generate a list of objects and link them to a room"""

        for room in self.map:
            self.map[room]["objects"] = list()
            self.map[room]["objects"].append("Door")
        for room in self.map:
            iter = len(self.rooms) - 1
            while iter > 0:
                if random.randrange(0, 2) > 0:
                    randObj = self.choose_object()
                    self.map[room]["objects"].append(randObj)
                iter = iter - 1
        for room in self.map:
            self.map[room]["objects"] = list(set(self.map[room]["objects"]))
        return self.map

    def check_room_objects(self, room):
        """Check what objects are in the room"""

        return self.map[room]["objects"]

    def check_room_connected(self, room):
        """Check what rooms are connected"""

        return self.map[room]["connected"]

    def remove_character_location(self, character):
        """Remove a characters location from memory"""
        for room in self.map:
            if character in self.map[room]["characters"]:
                  self.map[room]["characters"] = [x for x in self.map[room]["characters"] if x != character]
        return self.map

    def set_character_location(self, room, character):
        """Set a characters location in memory"""

        try:
           self.map[room]["characters"]
        except KeyError:
           for rooms in self.map:
               self.map[rooms]["characters"] = list()
        if type(self.map[room]["characters"]) != list:
            self.map[room]["characters"] = list()
        self.remove_character_location(character)
        self.map[room]["characters"].append(character)
        self.map[room]["characters"] = list(set(self.map[room]["characters"]))
        return self.map

    def check_character_location(self, character):
        """Get a characters location from memory"""
        for room in self.map:
            if character in self.map[room]["characters"]:
                return room
        # We use this return value to check if we need to assign a room to a character.
        return "Character Not Found"

    def choose_action(self):
        """Randomly choose an action to do"""

        self.actions.append("move")
        self.actions = list(set(self.actions))
        return self.actions[random.randrange(0, len(self.actions))]

    def choose_object_from_avail(self, room):
        """Randomly choose object from those available in the room"""

        available_objects = self.check_room_objects(room)
        return available_objects[random.randrange(0, len(available_objects))]

    def build_sentence(self):
        """Generate a sentence from available data"""
        # We choose what character we're going to write a sentence for.
        character_focus = self.choose_character()
        # We check if that character has a location yet, if not, give them one.
        if self.check_character_location(character_focus) == "Character Not Found":
            character_room = self.choose_room()
            self.set_character_location(character_room, character_focus)
        else:
            character_room = self.check_character_location(character_focus)
        # We check what action the character is going to take.
        character_action = self.choose_action()
        # If the character is moving, make sure they actually move.
        if character_action == "move":
            connected_rooms = self.check_room_connected(character_room)
            if len(connected_rooms) > 0:
                chosen_room = connected_rooms[random.randrange(0, len(connected_rooms))]
                self.set_character_location(chosen_room, character_focus)
                return " " + character_focus + " moved to the " + chosen_room + "."
            else:
                # If there's no connecting room, we random chance either teleporting or staying and crying.
                if random.randrange(0, 10) > 4:
                    chosen_room = self.rooms[random.randrange(0, len(self.rooms))]
                    self.set_character_location(chosen_room, character_focus)
                    return " " + character_focus + " teleported to the " + chosen_room + "."
                else:
                    return " " + character_focus + " cried because they were stuck."
        # If the character didn't move, we choose an object they will interact with.
        character_object = self.choose_object_from_avail(character_room)
        # We check if anybody else is in the room.
        character_avail = self.choose_character_from_avail(character_room, character_focus)
        # Set the noun based on whether they interact with a person or an object.
        if character_avail:
            if random.randrange(0, 10) > 4:
                available_chosen = character_avail
                noun = self.actionsDict[character_action].replace("the","")
            else:
                available_chosen = character_object
                noun = self.actionsDict[character_action] + " "
        else:
            available_chosen = character_object
            noun = self.actionsDict[character_action] + " "
        # We return a nicely built sentence.
        return " " + character_focus + " " + character_action.lower() + "ed " + noun + available_chosen + "."

    def choose_character_from_avail(self, room, character):
        """Return a random character who is also in the room."""

        characters_present = self.map[room]["characters"]
        available_list = list()
        for item in characters_present:
            if item != character:
                available_list.append(item)
        if len(available_list) > 0:
            return available_list[random.randrange(0, len(available_list))]
        else:
            return False

    def write_character_locations(self):
        """A nice paragraph intro that says where everyone is."""

        for i in self.characters:
            if self.check_character_location(i) == "Character Not Found":
                character_room = self.choose_room()
                self.set_character_location(character_room, i)
        return [x + " is in the " + self.check_character_location(x) + ".\n" for x in self.characters]

    def build_paragraph(self):
        """Take some paragraph intros and sentence and bind them together prettily."""

        paragraphIntro = str()
        for item in self.write_character_locations():
            paragraphIntro = paragraphIntro + item
        paragraph = str()
        paragraph_build = list()
        iter = 20
        while iter > 0:
            paragraph_build.append(self.build_sentence())
            iter = iter - 1
        for character in self.characters:
            list_iter = 0
            for item in paragraph_build:
                if character in item[:len(character)+1]:
                    paragraph = paragraph + paragraph_build[list_iter]
                list_iter = list_iter + 1
        return paragraphIntro + "\n" + paragraph[1:]

    def build_book(self):
        """Bind a bunch of paragraphs together with a rough estimate, to make a 50,000 word book"""

        bookStream = str()
        while len(bookStream) < (50000 * 8):
            bookStream = bookStream + self.build_paragraph() + "\n\n"
        return bookStream

    def write_book(self):
        """Write a whole 50,000 word book to file."""

        with open("outBook.md", "w+") as openFile:
            openFile.write(self.build_book())
        return "outBook.md"

if __name__ == "__main__":
    insane = CrayCrayWriter()
    insane.decide_map()
    insane.decide_room_objects()
    iter = 10
    while iter > 0:
        print(insane.build_paragraph())
        print("\n\n")
        iter = iter - 1
<|endoftext|>


<commit_before># Copyright 2016 Google Inc.  All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Classes representing the various compiler and linter tools that are used to
build Shaka Player."""

import json
import logging
import os
import re
import shutil
import subprocess

import generateLocalizations
import shakaBuildHelpers

def _canonicalize_source_files(source_files):
  """Canonicalize a set or list of source files.

  This makes all path names cygwin-safe and sorted."""

  files = [shakaBuildHelpers.cygwin_safe_path(f) for f in source_files]
  files.sort()
  return files

def _get_source_path(path):
  """Take path components of a source file as arguments and return an absolute,
     cygwin-safe path."""
  return shakaBuildHelpers.cygwin_safe_path(os.path.join(
      shakaBuildHelpers.get_source_base(), path))

def _must_build(output, source_files):
  """Returns True if any of the |source_files| have changed since |output| was
     built, or if |output| does not exist yet."""
  if not os.path.isfile(output):
    # Nothing built, so we should build the output.
    return True

  # Detect changes to the set of files that we intend to build.
  build_time = os.path.getmtime(output)
  # See if any files were modified since the output was created.
  if any(os.path.getmtime(f) > build_time for f in source_files):
    # Some input files have changed, so we should build again.
    return True

  logging.warning('No changes detected, skipping. Use --force to override.')
  return False

def _update_timestamp(path):
  # This creates the file if it does not exist, and updates the timestamp if it
  # does.
  open(path, 'w').close()

class ClosureCompiler(object):
  def __init__(self, source_files, build_name):
    self.source_files = _canonicalize_source_files(source_files)

    prefix = _get_source_path('dist/' + build_name)
    self.compiled_js_path = prefix + '.js'
    self.source_map_path = prefix + '.map'

    # These can be overridden for special cases:

    # If True, output the compiled bundle to a file.
    self.output_compiled_bundle = True
    # If True, generate a source map and attach it to the output bundle.
    self.add_source_map = True
    # If True, wrap the output in a wrapper that prevents window pollution.
    self.add_wrapper = True
    # If not None, use a timestamp file for change detection, and touch that
    # timestamp file after compilation.
    self.timestamp_file = None

  def compile(self, options, force=False):
    """Builds the files in |self.source_files| using the given Closure
    command-line options.

    Args:
      options: An array of options to give to Closure.
      force: Generate the output even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    if not force:
      if self.timestamp_file:
        if not _must_build(self.timestamp_file, self.source_files):
          return True
      else:
        if not _must_build(self.compiled_js_path, self.source_files):
          return True

    jar = _get_source_path('third_party/closure/compiler.jar')

    output_options = []
    if self.output_compiled_bundle:
      output_options += [
          '--js_output_file', self.compiled_js_path,
      ]

      if self.add_source_map:
        source_base = _get_source_path('')

        output_options += [
            '--create_source_map', self.source_map_path,
            # This uses a simple string replacement to create relative paths.
            # "source|replacement".
            '--source_map_location_mapping', source_base + '|../',
        ]
        if shakaBuildHelpers.is_windows() or shakaBuildHelpers.is_cygwin():
          output_options += [
              # On Windows, the source map needs to use '/' for paths, so we
              # need to have this mapping so it creates the correct relative
              # paths.  For some reason, we still need the mapping above for
              # other parts of the source map.
              '--source_map_location_mapping',
              source_base.replace('\\', '/') + '|../',
          ]

      if self.add_wrapper:
        output_options += self._prepare_wrapper()

    cmd_line = ['java', '-jar', jar] + output_options + options
    cmd_line += self.source_files

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      logging.error('Build failed')
      return False

    if self.output_compiled_bundle and self.add_source_map:
      # Add a special source-mapping comment so that Chrome and Firefox can map
      # line and character numbers from the compiled library back to the
      # original source locations.
      with open(self.compiled_js_path, 'a') as f:
        f.write('//# sourceMappingURL=%s' % os.path.basename(
            self.source_map_path))

    if self.timestamp_file:
      _update_timestamp(self.timestamp_file)

    return True

  def _prepare_wrapper(self):
    """Prepares an output wrapper and returns a list of command line arguments
       for Closure Compiler to use it."""

    # Load the wrapper and use Closure to strip whitespace and comments.
    # This requires %output% in the template to be protected, so Closure doesn't
    # fail to parse it.
    wrapper_input_path = _get_source_path('build/wrapper.template.js')
    wrapper_output_path = _get_source_path('dist/wrapper.js')

    with open(wrapper_input_path, 'rb') as f:
      wrapper_code = f.read().decode('utf8').replace('%output%', '"%output%"')

    jar = _get_source_path('third_party/closure/compiler.jar')
    cmd_line = ['java', '-jar', jar, '-O', 'WHITESPACE_ONLY']

    proc = shakaBuildHelpers.execute_subprocess(
        cmd_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    stripped_wrapper_code = proc.communicate(wrapper_code.encode('utf8'))[0]

    if proc.returncode != 0:
      raise RuntimeError('Failed to strip whitespace from wrapper!')

    with open(wrapper_output_path, 'wb') as f:
      code = stripped_wrapper_code.decode('utf8')
      f.write(code.replace('"%output%"', '%output%').encode('utf8'))

    return ['--output_wrapper_file=%s' % wrapper_output_path]

class ExternGenerator(object):
  def __init__(self, source_files, build_name):
    self.source_files = _canonicalize_source_files(source_files)
    self.output = _get_source_path('dist/' + build_name + '.externs.js')

  def generate(self, force=False):
    """Generates externs for the files in |self.source_files|.

    Args:
      force: Generate the output even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    if not force and not _must_build(self.output, self.source_files):
      return True

    extern_generator = _get_source_path('build/generateExterns.js')

    cmd_line = ['node', extern_generator, '--output', self.output]
    cmd_line += self.source_files

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      logging.error('Externs generation failed')
      return False

    return True

class Less(object):
  def __init__(self, main_source_file, all_source_files, output):
    # Less only takes one input file, but that input may import others.
    # We use main_source_file for compilation, but all_source_files to detect
    # if it needs to be rebuilt.
    self.main_source_file = _canonicalize_source_files([main_source_file])[0]
    self.all_source_files = _canonicalize_source_files(all_source_files)
    self.output = output

  def compile(self, force=False):
    """Compiles the main less file in |self.main_source_file| into the
       |self.output| css file.

    Args:
      force: Generate the output even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    if not force and not _must_build(self.output, self.all_source_files):
      return True

    lessc = shakaBuildHelpers.get_node_binary('less', 'lessc')

    cmd_line = lessc + [self.main_source_file, self.output]

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      logging.error('Externs generation failed')
      return False

    return True

class Linter(object):
  def __init__(self, source_files, config_path):
    self.source_files = _canonicalize_source_files(source_files)
    self.config_path = config_path
    self.output = _get_source_path('dist/.lintstamp')

  def lint(self, fix=False, force=False):
    """Run linter checks on the files in |self.source_files|.

    Args:
      fix: If True, ask the linter to fix what errors it can automatically.
      force: Run linter checks even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    deps = self.source_files + [self.config_path]
    if not force and not _must_build(self.output, deps):
      return True

    eslint = shakaBuildHelpers.get_node_binary('eslint')
    cmd_line = eslint + ['--config', self.config_path] + self.source_files

    if fix:
      cmd_line += ['--fix']

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      return False

    # TODO: Add back Closure Compiler Linter

    # Update the timestamp of the file that tracks when we last updated.
    _update_timestamp(self.output)
    return True

class HtmlLinter(object):
  def __init__(self, source_files, config_path):
    self.source_files = _canonicalize_source_files(source_files)
    self.config_path = config_path
    self.output = _get_source_path('dist/.htmllintstamp')

  def lint(self, force=False):
    """Run HTML linter checks on the files in |self.source_files|.

    Args:
      force: Run linter checks even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    deps = self.source_files + [self.config_path]
    if not force and not _must_build(self.output, deps):
      return True

    htmlhint = shakaBuildHelpers.get_node_binary('htmlhint')
    cmd_line = htmlhint + ['--config=' + self.config_path] + self.source_files

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      return False

    # Update the timestamp of the file that tracks when we last updated.
    _update_timestamp(self.output)
    return True

class Jsdoc(object):
  def __init__(self, config_path):
    self.config_path = config_path
    self.source_files = []

    # Just one of many output files, used to check the freshness of the docs.
    self.output = _get_source_path('docs/api/index.html')

    # To avoid getting out of sync with the source files jsdoc actually reads,
    # parse the config file and locate all source files based on that.
    match = re.compile(r'.*\.js$')
    with open(self.config_path, 'rb') as f:
      config = json.load(f)
    for path in config['source']['include']:
      full_path = _get_source_path(path)
      self.source_files += shakaBuildHelpers.get_all_files(full_path, match)

  def build(self, force=False):
    """Build the documentation.

    Args:
      force: Build the docs even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    deps = self.source_files + [self.config_path]
    if not force and not _must_build(self.output, deps):
      return True

    base = _get_source_path('')

    # Wipe out any old docs.
    shutil.rmtree(os.path.join(base, 'docs', 'api'), ignore_errors=True)

    # Jsdoc expects to run from the base dir.
    with shakaBuildHelpers.InDir(base):
      jsdoc = shakaBuildHelpers.get_node_binary('jsdoc')
      cmd_line = jsdoc + ['-c', self.config_path]
      if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
        return False

    return True

class GenerateLocalizations(object):
  def __init__(self, locales):
    self.locales = locales
    self.source_files = shakaBuildHelpers.get_all_files(
        _get_source_path('ui/locales'))
    self.output = _get_source_path('dist/locales.js')

  def _locales_changed(self):
    # If locales is None, it means we are being called by a caller who doesn't
    # care what locales are in use.  This is true, for example, when we are
    # running a compiler pass over the tests.
    if self.locales is None:
      return False

    # Find out what locales we used before.  If they have changed, we must
    # regenerate the output.
    last_locales = None
    try:
      prefix = '// LOCALES: '
      with open(self.output, 'r') as f:
        for line in f:
          if line.startswith(prefix):
            last_locales = line.replace(prefix, '').strip().split(', ')
    except IOError:
      # The file wasn't found or couldn't be read, so it needs to be redone.
      return True

    return set(last_locales) != set(self.locales)

  def generate(self, force=False):
    """Generate runtime localizations.

    Args:
      force: Generate the localizations even if the inputs and locales have not
             changed.

    Returns:
      True on success; False on failure.
    """

    if (not force and not _must_build(self.output, self.source_files) and
        not self._locales_changed()):
      return True

    locales = self.locales or ['en']
    generateLocalizations.main(['--locales'] + locales)
    return True
<commit_msg>Fix change detection for docs build<commit_after># Copyright 2016 Google Inc.  All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Classes representing the various compiler and linter tools that are used to
build Shaka Player."""

import json
import logging
import os
import re
import shutil
import subprocess

import generateLocalizations
import shakaBuildHelpers

def _canonicalize_source_files(source_files):
  """Canonicalize a set or list of source files.

  This makes all path names cygwin-safe and sorted."""

  files = [shakaBuildHelpers.cygwin_safe_path(f) for f in source_files]
  files.sort()
  return files

def _get_source_path(path):
  """Take path components of a source file as arguments and return an absolute,
     cygwin-safe path."""
  return shakaBuildHelpers.cygwin_safe_path(os.path.join(
      shakaBuildHelpers.get_source_base(), path))

def _must_build(output, source_files):
  """Returns True if any of the |source_files| have changed since |output| was
     built, or if |output| does not exist yet."""
  if not os.path.isfile(output):
    # Nothing built, so we should build the output.
    return True

  # Detect changes to the set of files that we intend to build.
  build_time = os.path.getmtime(output)
  # See if any files were modified since the output was created.
  if any(os.path.getmtime(f) > build_time for f in source_files):
    # Some input files have changed, so we should build again.
    return True

  logging.warning('No changes detected, skipping. Use --force to override.')
  return False

def _update_timestamp(path):
  # This creates the file if it does not exist, and updates the timestamp if it
  # does.
  open(path, 'w').close()

class ClosureCompiler(object):
  def __init__(self, source_files, build_name):
    self.source_files = _canonicalize_source_files(source_files)

    prefix = _get_source_path('dist/' + build_name)
    self.compiled_js_path = prefix + '.js'
    self.source_map_path = prefix + '.map'

    # These can be overridden for special cases:

    # If True, output the compiled bundle to a file.
    self.output_compiled_bundle = True
    # If True, generate a source map and attach it to the output bundle.
    self.add_source_map = True
    # If True, wrap the output in a wrapper that prevents window pollution.
    self.add_wrapper = True
    # If not None, use a timestamp file for change detection, and touch that
    # timestamp file after compilation.
    self.timestamp_file = None

  def compile(self, options, force=False):
    """Builds the files in |self.source_files| using the given Closure
    command-line options.

    Args:
      options: An array of options to give to Closure.
      force: Generate the output even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    if not force:
      if self.timestamp_file:
        if not _must_build(self.timestamp_file, self.source_files):
          return True
      else:
        if not _must_build(self.compiled_js_path, self.source_files):
          return True

    jar = _get_source_path('third_party/closure/compiler.jar')

    output_options = []
    if self.output_compiled_bundle:
      output_options += [
          '--js_output_file', self.compiled_js_path,
      ]

      if self.add_source_map:
        source_base = _get_source_path('')

        output_options += [
            '--create_source_map', self.source_map_path,
            # This uses a simple string replacement to create relative paths.
            # "source|replacement".
            '--source_map_location_mapping', source_base + '|../',
        ]
        if shakaBuildHelpers.is_windows() or shakaBuildHelpers.is_cygwin():
          output_options += [
              # On Windows, the source map needs to use '/' for paths, so we
              # need to have this mapping so it creates the correct relative
              # paths.  For some reason, we still need the mapping above for
              # other parts of the source map.
              '--source_map_location_mapping',
              source_base.replace('\\', '/') + '|../',
          ]

      if self.add_wrapper:
        output_options += self._prepare_wrapper()

    cmd_line = ['java', '-jar', jar] + output_options + options
    cmd_line += self.source_files

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      logging.error('Build failed')
      return False

    if self.output_compiled_bundle and self.add_source_map:
      # Add a special source-mapping comment so that Chrome and Firefox can map
      # line and character numbers from the compiled library back to the
      # original source locations.
      with open(self.compiled_js_path, 'a') as f:
        f.write('//# sourceMappingURL=%s' % os.path.basename(
            self.source_map_path))

    if self.timestamp_file:
      _update_timestamp(self.timestamp_file)

    return True

  def _prepare_wrapper(self):
    """Prepares an output wrapper and returns a list of command line arguments
       for Closure Compiler to use it."""

    # Load the wrapper and use Closure to strip whitespace and comments.
    # This requires %output% in the template to be protected, so Closure doesn't
    # fail to parse it.
    wrapper_input_path = _get_source_path('build/wrapper.template.js')
    wrapper_output_path = _get_source_path('dist/wrapper.js')

    with open(wrapper_input_path, 'rb') as f:
      wrapper_code = f.read().decode('utf8').replace('%output%', '"%output%"')

    jar = _get_source_path('third_party/closure/compiler.jar')
    cmd_line = ['java', '-jar', jar, '-O', 'WHITESPACE_ONLY']

    proc = shakaBuildHelpers.execute_subprocess(
        cmd_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    stripped_wrapper_code = proc.communicate(wrapper_code.encode('utf8'))[0]

    if proc.returncode != 0:
      raise RuntimeError('Failed to strip whitespace from wrapper!')

    with open(wrapper_output_path, 'wb') as f:
      code = stripped_wrapper_code.decode('utf8')
      f.write(code.replace('"%output%"', '%output%').encode('utf8'))

    return ['--output_wrapper_file=%s' % wrapper_output_path]

class ExternGenerator(object):
  def __init__(self, source_files, build_name):
    self.source_files = _canonicalize_source_files(source_files)
    self.output = _get_source_path('dist/' + build_name + '.externs.js')

  def generate(self, force=False):
    """Generates externs for the files in |self.source_files|.

    Args:
      force: Generate the output even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    if not force and not _must_build(self.output, self.source_files):
      return True

    extern_generator = _get_source_path('build/generateExterns.js')

    cmd_line = ['node', extern_generator, '--output', self.output]
    cmd_line += self.source_files

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      logging.error('Externs generation failed')
      return False

    return True

class Less(object):
  def __init__(self, main_source_file, all_source_files, output):
    # Less only takes one input file, but that input may import others.
    # We use main_source_file for compilation, but all_source_files to detect
    # if it needs to be rebuilt.
    self.main_source_file = _canonicalize_source_files([main_source_file])[0]
    self.all_source_files = _canonicalize_source_files(all_source_files)
    self.output = output

  def compile(self, force=False):
    """Compiles the main less file in |self.main_source_file| into the
       |self.output| css file.

    Args:
      force: Generate the output even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    if not force and not _must_build(self.output, self.all_source_files):
      return True

    lessc = shakaBuildHelpers.get_node_binary('less', 'lessc')

    cmd_line = lessc + [self.main_source_file, self.output]

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      logging.error('Externs generation failed')
      return False

    return True

class Linter(object):
  def __init__(self, source_files, config_path):
    self.source_files = _canonicalize_source_files(source_files)
    self.config_path = config_path
    self.output = _get_source_path('dist/.lintstamp')

  def lint(self, fix=False, force=False):
    """Run linter checks on the files in |self.source_files|.

    Args:
      fix: If True, ask the linter to fix what errors it can automatically.
      force: Run linter checks even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    deps = self.source_files + [self.config_path]
    if not force and not _must_build(self.output, deps):
      return True

    eslint = shakaBuildHelpers.get_node_binary('eslint')
    cmd_line = eslint + ['--config', self.config_path] + self.source_files

    if fix:
      cmd_line += ['--fix']

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      return False

    # TODO: Add back Closure Compiler Linter

    # Update the timestamp of the file that tracks when we last updated.
    _update_timestamp(self.output)
    return True

class HtmlLinter(object):
  def __init__(self, source_files, config_path):
    self.source_files = _canonicalize_source_files(source_files)
    self.config_path = config_path
    self.output = _get_source_path('dist/.htmllintstamp')

  def lint(self, force=False):
    """Run HTML linter checks on the files in |self.source_files|.

    Args:
      force: Run linter checks even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    deps = self.source_files + [self.config_path]
    if not force and not _must_build(self.output, deps):
      return True

    htmlhint = shakaBuildHelpers.get_node_binary('htmlhint')
    cmd_line = htmlhint + ['--config=' + self.config_path] + self.source_files

    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
      return False

    # Update the timestamp of the file that tracks when we last updated.
    _update_timestamp(self.output)
    return True

class Jsdoc(object):
  def __init__(self, config_path):
    self.config_path = config_path
    self.source_files = shakaBuildHelpers.get_all_files(
        _get_source_path('docs/tutorials'))
    self.source_files += shakaBuildHelpers.get_all_files(
        _get_source_path('docs/jsdoc-template'))
    self.source_files += [
        _get_source_path('docs/jsdoc-plugin.js'),
        _get_source_path('docs/api-mainpage.md'),
    ]

    # Just one of many output files, used to check the freshness of the docs.
    self.output = _get_source_path('docs/api/index.html')

    # To avoid getting out of sync with the source files jsdoc actually reads,
    # parse the config file and locate all source files based on that.
    match = re.compile(r'.*\.js$')
    with open(self.config_path, 'rb') as f:
      config = json.load(f)
    for path in config['source']['include']:
      full_path = _get_source_path(path)
      self.source_files += shakaBuildHelpers.get_all_files(full_path, match)

  def build(self, force=False):
    """Build the documentation.

    Args:
      force: Build the docs even if the inputs have not changed.

    Returns:
      True on success; False on failure.
    """
    deps = self.source_files + [self.config_path]
    if not force and not _must_build(self.output, deps):
      return True

    base = _get_source_path('')

    # Wipe out any old docs.
    shutil.rmtree(os.path.join(base, 'docs', 'api'), ignore_errors=True)

    # Jsdoc expects to run from the base dir.
    with shakaBuildHelpers.InDir(base):
      jsdoc = shakaBuildHelpers.get_node_binary('jsdoc')
      cmd_line = jsdoc + ['-c', self.config_path]
      if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
        return False

    return True

class GenerateLocalizations(object):
  def __init__(self, locales):
    self.locales = locales
    self.source_files = shakaBuildHelpers.get_all_files(
        _get_source_path('ui/locales'))
    self.output = _get_source_path('dist/locales.js')

  def _locales_changed(self):
    # If locales is None, it means we are being called by a caller who doesn't
    # care what locales are in use.  This is true, for example, when we are
    # running a compiler pass over the tests.
    if self.locales is None:
      return False

    # Find out what locales we used before.  If they have changed, we must
    # regenerate the output.
    last_locales = None
    try:
      prefix = '// LOCALES: '
      with open(self.output, 'r') as f:
        for line in f:
          if line.startswith(prefix):
            last_locales = line.replace(prefix, '').strip().split(', ')
    except IOError:
      # The file wasn't found or couldn't be read, so it needs to be redone.
      return True

    return set(last_locales) != set(self.locales)

  def generate(self, force=False):
    """Generate runtime localizations.

    Args:
      force: Generate the localizations even if the inputs and locales have not
             changed.

    Returns:
      True on success; False on failure.
    """

    if (not force and not _must_build(self.output, self.source_files) and
        not self._locales_changed()):
      return True

    locales = self.locales or ['en']
    generateLocalizations.main(['--locales'] + locales)
    return True
<|endoftext|>


<commit_before>import sys
import os
import time
import notedown
import nbformat

assert len(sys.argv) == 3, 'usage: input.md output.ipynb'

# timeout for each notebook, in sec
timeout = 20 * 60

# the files will be ingored for execution
ignore_execution = []

input_fn = sys.argv[1]
output_fn = sys.argv[2]

reader = notedown.MarkdownReader(match='strict')

do_eval = int(os.environ.get('EVAL', True))

# read
with open(input_fn, 'r') as f:
    notebook = reader.read(f)

if do_eval and not any([i in input_fn for i in ignore_execution]):
    tic = time.time()
    notedown.run(notebook, timeout)
    print('=== Finished evaluation in %f sec'%(time.time()-tic))

# write
# need to add language info to for syntax highlight
notebook['metadata'].update({'language_info':{'name':'python'}})

with open(output_fn, 'w') as f:
    f.write(nbformat.writes(notebook))
<commit_msg>add space between eng and zh words<commit_after>import sys
import os
import time
import notedown
import nbformat

assert len(sys.argv) == 3, 'usage: input.md output.ipynb'

def is_ascii(character):
    return ord(character) <= 128

def add_space_between_ascii_and_non_ascii(string):
    punc = (' ','\n','\t','\r','，','。','？','！','、','；','：','“',
            '”','（','）','【','】','—','…')
    if len(string) == 0:
        return ''
    ret = string[0]
    for i in range(1, len(string)):
        if ((is_ascii(string[i-1]) != is_ascii(string[i]))
            and (string[i-1] not in punc)
            and (string[i] not in punc)):
            ret += ' '
        ret += string[i]
    return ret
# timeout for each notebook, in sec
timeout = 20 * 60

# the files will be ingored for execution
ignore_execution = []

input_fn = sys.argv[1]
output_fn = sys.argv[2]

reader = notedown.MarkdownReader(match='strict')

do_eval = int(os.environ.get('EVAL', True))

# read
with open(input_fn, 'r') as f:
    notebook = reader.read(f)

for c in notebook.cells:
    c.source = add_space_between_ascii_and_non_ascii(c.source)

if do_eval and not any([i in input_fn for i in ignore_execution]):
    tic = time.time()
    notedown.run(notebook, timeout)
    print('=== Finished evaluation in %f sec'%(time.time()-tic))

# write
# need to add language info to for syntax highlight
notebook['metadata'].update({'language_info':{'name':'python'}})

with open(output_fn, 'w') as f:
    f.write(nbformat.writes(notebook))
<|endoftext|>


<commit_before>import numpy as np
import ipdb
import matplotlib.pylab as pl
pl.ion()
datadir = 'data/'
from astropy.io import fits
import cPickle as pickle
#import healpy as hp
from cosmolopy import distance, fidcosmo
cosmo = fidcosmo

def main():
    hemi='south'
    grid = grid3d(hemi=hemi)
    d_rand = load_sdss_randoms_both_catalogs(hemi)
    nrandom = grid.num_from_radecz(d_rand['ra'],d_rand['dec'], d_rand['z'])
    ipdb.set_trace()

class grid3d(object):
    def __init__(self, hemi='south', reso_mpc=8.0,
                 nx=2**9, ny=2**9, nz=2**9,
                 zmin=0.1, zmax=0.55):
        self.hemi = hemi
        self.reso_mpc = reso_mpc
        self.zmin = zmin
        self.zmax = zmax
        hemic = hemi.capitalize()
        lowz = fits.open(datadir+'galaxy_DR10v8_LOWZ_'+hemic+'.fits')[1].data
        cmass = fits.open(datadir+'galaxy_DR10v8_CMASS_'+hemic+'.fits')[1].data
        ra = np.concatenate([lowz['ra'], cmass['ra']])
        dec = np.concatenate([lowz['dec'], cmass['dec']])
        z = np.concatenate([lowz['z'], cmass['z']])
        wh = np.where((z>self.zmin)&(z<self.zmax))[0]
        ra=ra[wh]; dec=dec[wh]; z=z[wh]

        # do zmin/zmax in data loader functions?
        # insert here.#tmpp
        # CAREFUL, as digitize returns [0] for too-small values.

        rr = interp_comoving_distance(z)
        th = (90.-dec)*np.pi/180.
        phi = ra*np.pi/180.
        xx = rr*np.sin(th)*np.cos(phi)
        yy = rr*np.sin(th)*np.sin(phi)
        zz = rr*np.cos(th)
        self.nx=nx
        self.ny=ny
        self.nz=nz
        x1d = np.arange(nx)*reso_mpc
        y1d = np.arange(ny)*reso_mpc
        z1d = np.arange(nz)*reso_mpc
        for thing in [x1d, y1d, z1d]: thing -= np.mean(thing)
        x1d += xx.mean()
        y1d += yy.mean()
        z1d += zz.mean()
        self.x1d = x1d
        self.y1d = y1d
        self.z1d = z1d

    def get_voxel_indices(self, x, y, z):
        ind_x = np.digitize(x, x1d)
        ind_y = np.digitize(y, y1d)
        ind_z = np.digitize(z, z1d)
        return ind_x, ind_y, ind_z

    def num_from_xyz(self, x, y, z):
        n = np.zeros((self.nx, self.ny, self.nz))
        ind_x, ind_y, ind_z = get_voxel_indices(self, x, y, z)
        for xtmp,ytmp,ztmp in zip(ind_x, ind_y, ind_z):
            n[xtmp, ytmp, ztmp] += 1.
        return n

    def num_from_radecz(self, ra, dec, z):
        wh = np.where((z>self.zmin)&(z<self.zmax))[0]
        ra=ra[wh]; dec=dec[wh]; z=z[wh]
        rr = interp_comoving_distance(z)
        th = (90.-dec)*np.pi/180.
        phi = ra*np.pi/180.
        xx = rr*np.sin(th)*np.cos(phi)
        yy = rr*np.sin(th)*np.sin(phi)
        zz = rr*np.cos(th)
        n = num_from_xyz(xx, yy, zz)
        return n

def interp_comoving_distance(z, deltaz=1e-3):
    # It should be fine to do linear interpolation between points
    # separated by deltaz=1e-3.  The comoving distance function will be
    # very linear across that deltaz.  I checked and yes, agrees with
    # truth to ~1e-4 Mpc.
    zgrid = np.arange(np.min(z)-deltaz, np.max(z)+deltaz, deltaz)
    dgrid = distance.comoving_distance(zgrid, **cosmo)
    from scipy import interpolate
    f = interpolate.interp1d(zgrid, dgrid)
    return f(z)

def grid(hemisphere='south', zmin=0.1, zmax=0.55):
    lowz = fits.open(datadir+'galaxy_DR10v8_LOWZ_'+hemisphere.capitalize()+'.fits')[1].data
    cmass = fits.open(datadir+'galaxy_DR10v8_CMASS_'+hemisphere.capitalize()+'.fits')[1].data
    ra = np.concatenate([lowz['ra'], cmass['ra']])
    dec = np.concatenate([lowz['dec'], cmass['dec']])
    z = np.concatenate([lowz['z'], cmass['z']])
    wh = np.where((z>zmin)&(z<zmax))[0]
    ra=ra[wh]; dec=dec[wh]; z=z[wh]
    from cosmolopy import distance, fidcosmo
    print '...calculating distance...'
    rr = distance.comoving_distance(z, **fidcosmo)
    th = (90.-dec)*np.pi/180.
    phi = ra*np.pi/180.
    xx = rr*np.sin(th)*np.cos(phi)
    yy = rr*np.sin(th)*np.sin(phi)
    zz = rr*np.cos(th)

    reso_mpc = 8.0
    nx = 2**9
    ny = 2**9
    nz = 2**9

    x1d = np.arange(nx)*reso_mpc
    y1d = np.arange(ny)*reso_mpc
    z1d = np.arange(nz)*reso_mpc
    for thing in [x1d, y1d, z1d]: thing -= np.mean(thing)
    x1d += xx.mean()
    y1d += yy.mean()
    z1d += zz.mean()

    ind_x = np.digitize(xx, x1d)
    ind_y = np.digitize(yy, y1d)
    ind_z = np.digitize(zz, z1d)
    n = np.zeros((nx,ny,nz))
    for xtmp,ytmp,ztmp in zip(ind_x, ind_y, ind_z):
        n[xtmp, ytmp, ztmp] += 1.

    pl.clf()
    for i in range(3):
        pl.subplot(1,3,i+1)
        pl.imshow(n.sum(i))
        if i==0: pl.title('%s, %.1f Mpc resolution'%(hemisphere,reso_mpc))

    for thing in [xx,yy,zz]:
        print thing.min(), thing.max(), thing.max()-thing.min(), np.log2((thing.max()-thing.min())/reso_mpc)

    ipdb.set_trace()

def download_redmapper():
    from urllib import urlretrieve
    basepath = 'http://slac.stanford.edu/~erykoff/redmapper/catalogs/'
    file = 'redmapper_dr8_public_v5.2_catalog.fits'
    url = basepath + file
    savename = datadir + file
    urlretrieve(url, savename)

def download_lowz_and_cmass():
    from os import system
    from urllib import urlretrieve
    files=['galaxy_DR10v8_LOWZ_North.fits','galaxy_DR10v8_LOWZ_South.fits',
           'galaxy_DR10v8_CMASS_North.fits','galaxy_DR10v8_CMASS_South.fits',
           'random1_DR10v8_CMASS_North.fits', 'random1_DR10v8_CMASS_South.fits',
           'random1_DR10v8_LOWZ_North.fits', 'random1_DR10v8_LOWZ_South.fits',
           'random2_DR10v8_CMASS_North.fits', 'random2_DR10v8_CMASS_South.fits',
           'random2_DR10v8_LOWZ_North.fits', 'random2_DR10v8_LOWZ_South.fits']
    basepath = 'http://data.sdss3.org/sas/dr10/boss/lss/'
    for file in files:
        url = basepath + file + '.gz'
        savename = datadir + file + '.gz'
        urlretrieve(url, savename)
        system('gunzip '+savename)

def load_sdss_randoms(catalog, hemi):
    catup = catalog.upper()
    hemic = hemi.capitalize()
    ra = []; dec=[]; z=[]
    for i in range(2):
        filename = datadir+'random%i_DR10v8_'%(i+1)+catup+'_'+hemic+'.fits'
        tmp = fits.open(filename)[1].data
        ra.append(tmp['ra'])
        dec.append(tmp['dec'])
        z.append(tmp['z'])
    ra = np.hstack(ra)
    dec = np.hstack(dec)
    z = np.hstack(z)
    return {'ra':ra, 'dec':dec, 'z':z}

def load_sdss_data(catalog, hemi):
    catup = catalog.upper()
    hemic = hemi.capitalize()
    filename = datadir+'galaxy_DR10v8_'+catup+'_'+hemic+'.fits'
    tmp = fits.open(filename)[1].data
    output = {'ra':tmp['ra'], 'dec':tmp['dec'], 'z':tmp['z']}
    return output

def load_sdss_randoms_both_catalogs(hemi):
    lowz = load_sdss_randoms('lowz', hemi)
    cmass = load_sdss_randoms('cmass', hemi)
    ra = np.hstack([lowz['ra'],cmass['ra']])
    dec = np.hstack([lowz['dec'],cmass['dec']])
    z = np.hstack([lowz['z'],cmass['z']])
    return {'ra':ra, 'dec':dec, 'z':z}

def load_sdss_data_both_catalogs(hemi):
    lowz = load_sdss_data('lowz', hemi)
    cmass = load_sdss_data('cmass', hemi)
    ra = np.hstack([lowz['ra'],cmass['ra']])
    dec = np.hstack([lowz['dec'],cmass['dec']])
    z = np.hstack([lowz['z'],cmass['z']])
    return {'ra':ra, 'dec':dec, 'z':z}

def do_everything():
    download_redmapper()
    download_lowz_and_cmass()

<commit_msg>made delta<commit_after>import numpy as np
import ipdb
import matplotlib.pylab as pl
pl.ion()
datadir = 'data/'
from astropy.io import fits
import cPickle as pickle
#import healpy as hp
from cosmolopy import distance, fidcosmo
cosmo = fidcosmo

def main( hemi='south'):
    grid = grid3d(hemi=hemi)
    d_data = load_sdss_data_both_catalogs(hemi)
    n_data = grid.num_from_radecz(d_data['ra'],d_data['dec'], d_data['z'])
    d_rand = load_sdss_randoms_both_catalogs(hemi)
    n_rand = grid.num_from_radecz(d_rand['ra'],d_rand['dec'], d_rand['z'])
    n_rand *= (1.*n_data.sum()/n_rand.sum())
    from scipy.ndimage import gaussian_filter
    fwhm_sm = 2.
    sigma_sm = fwhm_sm/2.355
    # tmpp, replace with a wiener filter that knows about LSS and shot noise.
    n_data = gaussian_filter(n_data, sigma_sm)
    n_rand = gaussian_filter(n_rand, sigma_sm)
    delta = (n_data-n_rand)/n_rand
    delta[n_rand==0]=0.
    delta_max = 3.
    delta[delta>delta_max]=delta_max
    pl.clf(); pl.imshow(delta[:,:,140])
    ipdb.set_trace()

class grid3d(object):
    def __init__(self, hemi='south', reso_mpc=16.0,
                 nx=2**8, ny=2**8, nz=2**8,
                 zmin=0.1, zmax=0.55):
        self.hemi = hemi
        self.reso_mpc = reso_mpc
        self.zmin = zmin
        self.zmax = zmax
        self.nx=nx
        self.ny=ny
        self.nz=nz
        d = load_sdss_data_both_catalogs(hemi)
        ra = d['ra']; dec=d['dec']; z=d['z']
        wh = np.where((z>self.zmin)&(z<self.zmax))[0]
        ra=ra[wh]; dec=dec[wh]; z=z[wh]
        rr = interp_comoving_distance(z)
        th = (90.-dec)*np.pi/180.
        phi = ra*np.pi/180.
        xx = rr*np.sin(th)*np.cos(phi)
        yy = rr*np.sin(th)*np.sin(phi)
        zz = rr*np.cos(th)
        x1d = np.arange(nx)*reso_mpc
        y1d = np.arange(ny)*reso_mpc
        z1d = np.arange(nz)*reso_mpc
        for thing in [x1d, y1d, z1d]: thing -= np.mean(thing)
        x1d += xx.mean()
        y1d += yy.mean()
        z1d += zz.mean()
        self.x1d = x1d
        self.y1d = y1d
        self.z1d = z1d

    def get_voxel_indices(self, x, y, z):
        ind_x = np.digitize(x, self.x1d)
        ind_y = np.digitize(y, self.y1d)
        ind_z = np.digitize(z, self.z1d)
        return ind_x, ind_y, ind_z

    def num_from_xyz(self, x, y, z):
        n = np.zeros((self.nx, self.ny, self.nz))
        ind_x, ind_y, ind_z = self.get_voxel_indices(x, y, z)
        for xtmp,ytmp,ztmp in zip(ind_x, ind_y, ind_z):
            n[xtmp, ytmp, ztmp] += 1.
        return n

    def num_from_radecz(self, ra, dec, z):
        wh = np.where((z>self.zmin)&(z<self.zmax))[0]
        ra=ra[wh]; dec=dec[wh]; z=z[wh]
        rr = interp_comoving_distance(z)
        th = (90.-dec)*np.pi/180.
        phi = ra*np.pi/180.
        xx = rr*np.sin(th)*np.cos(phi)
        yy = rr*np.sin(th)*np.sin(phi)
        zz = rr*np.cos(th)
        n = self.num_from_xyz(xx, yy, zz)
        return n

def interp_comoving_distance(z, deltaz=1e-3):
    # It should be fine to do linear interpolation between points
    # separated by deltaz=1e-3.  The comoving distance function will be
    # very linear across that deltaz.  I checked and yes, agrees with
    # truth to ~1e-4 Mpc.
    zgrid = np.arange(np.min(z)-deltaz, np.max(z)+deltaz, deltaz)
    dgrid = distance.comoving_distance(zgrid, **cosmo)
    from scipy import interpolate
    f = interpolate.interp1d(zgrid, dgrid)
    return f(z)

def grid(hemisphere='south', zmin=0.1, zmax=0.55):
    lowz = fits.open(datadir+'galaxy_DR10v8_LOWZ_'+hemisphere.capitalize()+'.fits')[1].data
    cmass = fits.open(datadir+'galaxy_DR10v8_CMASS_'+hemisphere.capitalize()+'.fits')[1].data
    ra = np.concatenate([lowz['ra'], cmass['ra']])
    dec = np.concatenate([lowz['dec'], cmass['dec']])
    z = np.concatenate([lowz['z'], cmass['z']])
    wh = np.where((z>zmin)&(z<zmax))[0]
    ra=ra[wh]; dec=dec[wh]; z=z[wh]
    from cosmolopy import distance, fidcosmo
    print '...calculating distance...'
    rr = distance.comoving_distance(z, **fidcosmo)
    th = (90.-dec)*np.pi/180.
    phi = ra*np.pi/180.
    xx = rr*np.sin(th)*np.cos(phi)
    yy = rr*np.sin(th)*np.sin(phi)
    zz = rr*np.cos(th)

    reso_mpc = 8.0
    nx = 2**9
    ny = 2**9
    nz = 2**9

    x1d = np.arange(nx)*reso_mpc
    y1d = np.arange(ny)*reso_mpc
    z1d = np.arange(nz)*reso_mpc
    for thing in [x1d, y1d, z1d]: thing -= np.mean(thing)
    x1d += xx.mean()
    y1d += yy.mean()
    z1d += zz.mean()

    ind_x = np.digitize(xx, x1d)
    ind_y = np.digitize(yy, y1d)
    ind_z = np.digitize(zz, z1d)
    n = np.zeros((nx,ny,nz))
    for xtmp,ytmp,ztmp in zip(ind_x, ind_y, ind_z):
        n[xtmp, ytmp, ztmp] += 1.

    pl.clf()
    for i in range(3):
        pl.subplot(1,3,i+1)
        pl.imshow(n.sum(i))
        if i==0: pl.title('%s, %.1f Mpc resolution'%(hemisphere,reso_mpc))

    for thing in [xx,yy,zz]:
        print thing.min(), thing.max(), thing.max()-thing.min(), np.log2((thing.max()-thing.min())/reso_mpc)

    ipdb.set_trace()

def download_redmapper():
    from urllib import urlretrieve
    basepath = 'http://slac.stanford.edu/~erykoff/redmapper/catalogs/'
    file = 'redmapper_dr8_public_v5.2_catalog.fits'
    url = basepath + file
    savename = datadir + file
    urlretrieve(url, savename)

def download_lowz_and_cmass():
    from os import system
    from urllib import urlretrieve
    files=['galaxy_DR10v8_LOWZ_North.fits','galaxy_DR10v8_LOWZ_South.fits',
           'galaxy_DR10v8_CMASS_North.fits','galaxy_DR10v8_CMASS_South.fits',
           'random1_DR10v8_CMASS_North.fits', 'random1_DR10v8_CMASS_South.fits',
           'random1_DR10v8_LOWZ_North.fits', 'random1_DR10v8_LOWZ_South.fits',
           'random2_DR10v8_CMASS_North.fits', 'random2_DR10v8_CMASS_South.fits',
           'random2_DR10v8_LOWZ_North.fits', 'random2_DR10v8_LOWZ_South.fits']
    basepath = 'http://data.sdss3.org/sas/dr10/boss/lss/'
    for file in files:
        url = basepath + file + '.gz'
        savename = datadir + file + '.gz'
        urlretrieve(url, savename)
        system('gunzip '+savename)

def load_sdss_randoms(catalog, hemi):
    catup = catalog.upper()
    hemic = hemi.capitalize()
    ra = []; dec=[]; z=[]
    for i in range(2):
        filename = datadir+'random%i_DR10v8_'%(i+1)+catup+'_'+hemic+'.fits'
        tmp = fits.open(filename)[1].data
        ra.append(tmp['ra'])
        dec.append(tmp['dec'])
        z.append(tmp['z'])
    ra = np.hstack(ra)
    dec = np.hstack(dec)
    z = np.hstack(z)
    return {'ra':ra, 'dec':dec, 'z':z}

def load_sdss_data(catalog, hemi):
    catup = catalog.upper()
    hemic = hemi.capitalize()
    filename = datadir+'galaxy_DR10v8_'+catup+'_'+hemic+'.fits'
    tmp = fits.open(filename)[1].data
    output = {'ra':tmp['ra'], 'dec':tmp['dec'], 'z':tmp['z']}
    return output

def load_sdss_randoms_both_catalogs(hemi):
    lowz = load_sdss_randoms('lowz', hemi)
    cmass = load_sdss_randoms('cmass', hemi)
    ra = np.hstack([lowz['ra'],cmass['ra']])
    dec = np.hstack([lowz['dec'],cmass['dec']])
    z = np.hstack([lowz['z'],cmass['z']])
    return {'ra':ra, 'dec':dec, 'z':z}

def load_sdss_data_both_catalogs(hemi):
    lowz = load_sdss_data('lowz', hemi)
    cmass = load_sdss_data('cmass', hemi)
    ra = np.hstack([lowz['ra'],cmass['ra']])
    dec = np.hstack([lowz['dec'],cmass['dec']])
    z = np.hstack([lowz['z'],cmass['z']])
    return {'ra':ra, 'dec':dec, 'z':z}

def do_everything():
    download_redmapper()
    download_lowz_and_cmass()

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import tensorflow.contrib.slim as slim
from utils import leaky_relu

def dcgan_generator(inputs, scope, reuse=None, output_height=64,
                    output_width=None, fc1_c=1024, grayscale=False):
  if not output_width:
    output_width = output_height

  with tf.variable_scope(scope, 'Generator', [inputs], reuse=reuse):
    fc1_h = output_height // 16
    fc1_w = output_width // 16
    if fc1_h == 0 or fc1_w == 0:
      raise ValueError('The output of generator is too small, at least 16*16')
    if fc1_c // 8 == 0:
      raise ValueError('The channel of Tensor fc1 is too small, at least 8')
    if grayscale:
      output_c = 1
    else:
      output_c = 3
    fc1 = slim.fully_connected(inputs=inputs,
                               num_outputs=fc1_h*fc1_w*fc1_c,
                               activation_fn=None,
                               scope='fc1')
    fc1_reshape = tf.reshape(fc1, [-1, fc1_h, fc1_w, fc1_c])
    bn1 = slim.batch_norm(inputs=fc1_reshape, activation_fn=tf.nn.relu, scope='bn1')
    deconv2d2 = slim.conv2d_transpose(inputs=bn1,
                                      num_outputs=fc1_c//2,
                                      kernel_size=5,
                                      stride=2,
                                      padding='SAME',
                                      activation_fn=tf.nn.relu,
                                      normalizer_fn=slim.batch_norm,
                                      scope='deconv2d2')
    print(deconv2d2.shape)
    deconv2d3 = slim.conv2d_transpose(inputs=deconv2d2,
                                      num_outputs=fc1_c//4,
                                      kernel_size=5,
                                      stride=2,
                                      padding='SAME',
                                      activation_fn=tf.nn.relu,
                                      normalizer_fn=slim.batch_norm,
                                      scope='deconv2d3')

    deconv2d4 = slim.conv2d_transpose(inputs=deconv2d3,
                                      num_outputs=fc1_c//8,
                                      kernel_size=5,
                                      stride=2,
                                      padding='SAME',
                                      activation_fn=tf.nn.relu,
                                      normalizer_fn=slim.batch_norm,
                                      scope='deconv2d4')
    deconv2d5 = slim.conv2d_transpose(inputs=deconv2d4,
                                      num_outputs=output_c,
                                      kernel_size=5,
                                      stride=2,
                                      padding='SAME',
                                      activation_fn=tf.tanh,
                                      normalizer_fn=None,
                                      scope='deconv2d5')
  return deconv2d5

def dcgan_discriminator(inputs, scope, reuse=None, conv2d1_c=128,
                        grayscale=False):
  if grayscale:
    if inputs.shape.ndims == 3:
      inputs = tf.expand_dims(inputs, -1)
    elif inputs.shape.ndims != 4 or inputs.shape[3] != 1:
      raise ValueError('Inputs shape is invalid for grayscale images, shape is ', inputs.shape.dims)
  else:
    if inputs.shape.dims != 4 or inputs.shape[3] != 3:
      raise ValueError('Inputs shape is invalid for normal images, shape is ', inputs.shape)

  with tf.variable_scope(scope, 'Discriminator', [inputs], reuse=reuse):
    conv2d1 = slim.conv2d(inputs=inputs,
                          num_outputs=conv2d1_c,
                          kernel_size=5,
                          stride=2,
                          padding='SAME',
                          activation_fn=leaky_relu,
                          normalizer_fn=None,
                          scope='conv2d1')
    conv2d2 = slim.conv2d(inputs=conv2d1,
                          num_outputs=conv2d1_c * 2,
                          kernel_size=5,
                          stride=2,
                          padding='SAME',
                          activation_fn=leaky_relu,
                          normalizer_fn=slim.batch_norm,
                          scope='conv2d2')

    conv2d3 = slim.conv2d(inputs=conv2d2,
                          num_outputs=conv2d1_c * 4,
                          kernel_size=5,
                          stride=2,
                          padding='SAME',
                          activation_fn=leaky_relu,
                          normalizer_fn=slim.batch_norm,
                          scope='conv2d3')

    conv2d4 = slim.conv2d(inputs=conv2d3,
                          num_outputs=conv2d1_c * 8,
                          kernel_size=5,
                          stride=2,
                          padding='SAME',
                          activation_fn=leaky_relu,
                          normalizer_fn=slim.batch_norm,
                          scope='conv2d4')
    flatten5 = slim.flatten(inputs=conv2d4, scope='flatten5')
    fc5 = slim.fully_connected(inputs=flatten5,
                               num_outputs=1,
                               activation_fn=tf.sigmoid,
                               scope='fc5')
    fc5 = tf.squeeze(fc5)
  return fc5

<commit_msg>Fix conv2d_transpose output shape bug by replacing tf.contrib.layers.conv2d_transpose by more basic tf.nn.conv2d_transpose<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math

import tensorflow as tf
import tensorflow.contrib.slim as slim
from utils import leaky_relu

def conv2d_transpose(inputs, output_shape, kernel_size, stride, padding, activation_fn, normalizer_fn, scope):
  with tf.variable_scope(scope, 'conv2d_transpose'):
    filter_ = tf.get_variable('filter', [kernel_size, kernel_size, output_shape[3], inputs.shape[3]],
                              trainable=True,
                              collections=[tf.GraphKeys.TRAINABLE_VARIABLES,
                                           tf.GraphKeys.MODEL_VARIABLES,
                                           tf.GraphKeys.GLOBAL_VARIABLES],
                              dtype=tf.float32)
    conv_t = tf.nn.conv2d_transpose(value=inputs,
                                    filter=filter_,
                                    output_shape=output_shape,
                                    strides=[1, stride, stride, 1],
                                    padding=padding,
                                    name=scope)
    if normalizer_fn:
      conv_t = normalizer_fn(conv_t)
    if activation_fn:
      conv_t = activation_fn(conv_t)

  return conv_t

def dcgan_generator(inputs, scope, reuse=None, output_height=64,
                    output_width=None, fc1_c=1024, grayscale=False):
  if not output_width:
    output_width = output_height

  with tf.variable_scope(scope, 'Generator', [inputs], reuse=reuse):
    fc1_h = int(math.ceil(output_height / 16.0))
    fc1_w = int(math.ceil(output_width / 16.0))
    deconv2d2_h = int(math.ceil(output_height / 8.0))
    deconv2d2_w = int(math.ceil(output_width / 8.0))
    deconv2d3_h = int(math.ceil(output_height / 4.0))
    deconv2d3_w = int(math.ceil(output_width / 4.0))
    deconv2d4_h = int(math.ceil(output_height / 2.0))
    deconv2d4_w = int(math.ceil(output_width / 2.0))
    batch_size = inputs.shape[0].value

    if fc1_h == 0 or fc1_w == 0:
      raise ValueError('Now the code should not reach here fc1_h and fc1_w should not be 0')
    if fc1_c // 8 == 0:
      raise ValueError('The channel of Tensor fc1 is too small, at least 8')
    if grayscale:
      output_c = 1
    else:
      output_c = 3
    fc1 = slim.fully_connected(inputs=inputs,
                               num_outputs=fc1_h*fc1_w*fc1_c,
                               activation_fn=None,
                               scope='fc1')
    fc1_reshape = tf.reshape(fc1, [-1, fc1_h, fc1_w, fc1_c])
    bn1 = slim.batch_norm(inputs=fc1_reshape, activation_fn=tf.nn.relu, scope='bn1')
    print(bn1.shape)
    deconv2d2 = conv2d_transpose(inputs=bn1,
                                 output_shape=[batch_size, deconv2d2_h, deconv2d2_w, fc1_c // 2],
                                 kernel_size=5,
                                 stride=2,
                                 padding='SAME',
                                 activation_fn=tf.nn.relu,
                                 normalizer_fn=slim.batch_norm,
                                 scope='deconv2d2')
    print(deconv2d2.shape)
    deconv2d3 = conv2d_transpose(inputs=deconv2d2,
                                 output_shape=[batch_size, deconv2d3_h, deconv2d3_w, fc1_c // 4],
                                 kernel_size=5,
                                 stride=2,
                                 padding='SAME',
                                 activation_fn=tf.nn.relu,
                                 normalizer_fn=slim.batch_norm,
                                 scope='deconv2d3')
    print(deconv2d3.shape)
    deconv2d4 = conv2d_transpose(inputs=deconv2d3,
                                 output_shape=[batch_size, deconv2d4_h, deconv2d4_w, fc1_c // 8],
                                 kernel_size=5,
                                 stride=2,
                                 padding='SAME',
                                 activation_fn=tf.nn.relu,
                                 normalizer_fn=slim.batch_norm,
                                 scope='deconv2d4')
    deconv2d5 = conv2d_transpose(inputs=deconv2d4,
                                 output_shape=[batch_size, output_height, output_width, output_c],
                                 kernel_size=5,
                                 stride=2,
                                 padding='SAME',
                                 activation_fn=tf.tanh,
                                 normalizer_fn=None,
                                 scope='deconv2d5')

  return deconv2d5

def dcgan_discriminator(inputs, scope, reuse=None, conv2d1_c=128,
                        grayscale=False):
  if grayscale:
    if inputs.shape.ndims == 3:
      inputs = tf.expand_dims(inputs, -1)
    elif inputs.shape.ndims != 4 or inputs.shape[3] != 1:
      raise ValueError('Inputs shape is invalid for grayscale images, shape is ', inputs.shape.dims)
  else:
    if inputs.shape.dims != 4 or inputs.shape[3] != 3:
      raise ValueError('Inputs shape is invalid for normal images, shape is ', inputs.shape)

  with tf.variable_scope(scope, 'Discriminator', [inputs], reuse=reuse):
    conv2d1 = slim.conv2d(inputs=inputs,
                          num_outputs=conv2d1_c,
                          kernel_size=5,
                          stride=2,
                          padding='SAME',
                          activation_fn=leaky_relu,
                          normalizer_fn=None,
                          scope='conv2d1')
    conv2d2 = slim.conv2d(inputs=conv2d1,
                          num_outputs=conv2d1_c * 2,
                          kernel_size=5,
                          stride=2,
                          padding='SAME',
                          activation_fn=leaky_relu,
                          normalizer_fn=slim.batch_norm,
                          scope='conv2d2')

    conv2d3 = slim.conv2d(inputs=conv2d2,
                          num_outputs=conv2d1_c * 4,
                          kernel_size=5,
                          stride=2,
                          padding='SAME',
                          activation_fn=leaky_relu,
                          normalizer_fn=slim.batch_norm,
                          scope='conv2d3')

    conv2d4 = slim.conv2d(inputs=conv2d3,
                          num_outputs=conv2d1_c * 8,
                          kernel_size=5,
                          stride=2,
                          padding='SAME',
                          activation_fn=leaky_relu,
                          normalizer_fn=slim.batch_norm,
                          scope='conv2d4')
    flatten5 = slim.flatten(inputs=conv2d4, scope='flatten5')
    fc5 = slim.fully_connected(inputs=flatten5,
                               num_outputs=1,
                               activation_fn=tf.sigmoid,
                               scope='fc5')
    fc5 = tf.squeeze(fc5)
  return fc5

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2012 Cyril Bonté
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
from optparse import OptionParser

from parser import converter, git_parser

def main():
    """
    Main script to execute the conversion process
    """

    usage = "Usage: %prog [options] file..."

    optparser = OptionParser(
        description='Generate HTML Document from HAProxy configuration.txt',
        usage=usage
    )
    optparser.add_option(
        '--git-directory', '-g',
        help='Optional git directory for input files, '
        'to determine haproxy details'
    )
    optparser.add_option(
        '--output-directory', '-o', default='.',
        help='Destination directory to store files, '
        'instead of the current working directory'
    )
    optparser.add_option(
        '--base', '-b', default='', help='Base directory for relative links'
    )
    (option, files) = optparser.parse_args()

    if not files:
        optparser.print_help()
        exit(1)

    option.output_directory = os.path.abspath(option.output_directory)
    if option.git_directory:
        option.git_directory = os.path.abspath(option.git_directory)

    script_place = os.path.dirname(__file__)
    os.chdir(script_place if script_place else '.')
    converter.convert_all(files, option.output_directory, option.base)

if __name__ == '__main__':
    main()
<commit_msg>remove unused import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2012 Cyril Bonté
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from optparse import OptionParser

from parser import converter, git_parser

def main():
    """
    Main script to execute the conversion process
    """

    usage = "Usage: %prog [options] file..."

    optparser = OptionParser(
        description='Generate HTML Document from HAProxy configuration.txt',
        usage=usage
    )
    optparser.add_option(
        '--git-directory', '-g',
        help='Optional git directory for input files, '
        'to determine haproxy details'
    )
    optparser.add_option(
        '--output-directory', '-o', default='.',
        help='Destination directory to store files, '
        'instead of the current working directory'
    )
    optparser.add_option(
        '--base', '-b', default='', help='Base directory for relative links'
    )
    (option, files) = optparser.parse_args()

    if not files:
        optparser.print_help()
        exit(1)

    option.output_directory = os.path.abspath(option.output_directory)
    if option.git_directory:
        option.git_directory = os.path.abspath(option.git_directory)

    script_place = os.path.dirname(__file__)
    os.chdir(script_place if script_place else '.')
    converter.convert_all(files, option.output_directory, option.base)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2

import ConfigParser, os, re

def init_wireless(dev):
    os.system('ip link set ' + dev + ' down')
    os.system("rfkill unblock wifi")
    os.system('killall -9 dhclient')
    os.system('killall -9 wpa_supplicant')
    os.system('ip link set ' + dev + ' up')
    os.system('sleep 1')

def connect_wep(dev, ssid):
    os.system("ifconfig " + dev + " down")
    os.system("iwconfig " + dev + " essid '" + ssid + "'")
    os.system("iwconfig " + dev + " ap any")
    os.system('ip link set ' + dev + ' up')

def connect_wpa(dev, ssid, psk):
    os.system("wpa_passphrase '" + ssid  + "' '" + psk + "' > /etc/wpa_supplicant/wpa_supplicant.conf")
    os.system("wpa_supplicant -d -Dwext  -i " + dev + " -c /etc/wpa_supplicant/wpa_supplicant.conf >& /var/tmp/wpa_supplicant.log &")
    os.system("sleep 5")

def set_dns(dns1, dns2):
    fd = open('/etc/resolv.conf', 'w')
    fd.write('nameserver ' + dns1 + '\n')
    fd.write('nameserver ' + dns2 + '\n')
    fd.close()

config = ConfigParser.ConfigParser()
config.readfp(open('/etc/vsnm.cfg'))

dev = config.get('global', 'dev');
init_wireless(dev)

p = os.popen('iwlist ' + dev + ' scan');
ssid_list = []
while 1:
    line = p.readline()
    if not line:
        break;
    if line.find("ESSID") != -1:
        line = re.sub("^.*ESSID:", "", line)
        line = re.sub('"', '', line)
        line = line.replace('\n', '')
        ssid_list.append(line)
        print(line)

for entry in config.sections():
    if entry != 'global':
        ssid = config.get(entry, 'ssid')
        try:
            i = ssid_list.index(ssid)
        except ValueError:
            continue

        protocol = config.get(entry, 'protocol')

        if 'WEP' == protocol:
            connect_wep(dev, ssid)
        elif 'WPA' == protocol:
            psk = config.get(entry, 'psk')
            print("ssid: " + ssid)
            print("psk: " + psk)
            connect_wpa(dev, ssid, psk)
        else:
            print("ERROR: protocol not recognized")

	net = config.get(entry, 'net')
	if 'dhcp' == net:
	    os.system("dhclient " + dev)
	elif 'static' == net:
	    address = config.get(entry, 'address')
        gateway = config.get(entry, 'gateway')
        print("address: " + address)
        print("gateway: " + gateway)
        os.system("ifconfig " + dev + " " + address)
        os.system("route add default gw " + gateway + " " + dev)
        try:
            os.system("sh " + config.get(entry, 'post-script'))
        except ConfigParser.NoOptionError:
            break
<commit_msg>fix identation<commit_after>#!/usr/bin/env python2

import ConfigParser, os, re

def init_wireless(dev):
    os.system('ip link set ' + dev + ' down')
    os.system("rfkill unblock wifi")
    os.system('killall -9 dhclient')
    os.system('killall -9 wpa_supplicant')
    os.system('ip link set ' + dev + ' up')
    os.system('sleep 1')

def connect_wep(dev, ssid):
    os.system("ifconfig " + dev + " down")
    os.system("iwconfig " + dev + " essid '" + ssid + "'")
    os.system("iwconfig " + dev + " ap any")
    os.system('ip link set ' + dev + ' up')

def connect_wpa(dev, ssid, psk):
    os.system("wpa_passphrase '" + ssid  + "' '" + psk + "' > /etc/wpa_supplicant/wpa_supplicant.conf")
    os.system("wpa_supplicant -d -Dwext  -i " + dev + " -c /etc/wpa_supplicant/wpa_supplicant.conf >& /var/tmp/wpa_supplicant.log &")
    os.system("sleep 5")

def set_dns(dns1, dns2):
    fd = open('/etc/resolv.conf', 'w')
    fd.write('nameserver ' + dns1 + '\n')
    fd.write('nameserver ' + dns2 + '\n')
    fd.close()

config = ConfigParser.ConfigParser()
config.readfp(open('/etc/vsnm.cfg'))

dev = config.get('global', 'dev');
init_wireless(dev)

p = os.popen('iwlist ' + dev + ' scan');
ssid_list = []
while 1:
    line = p.readline()
    if not line:
        break;
    if line.find("ESSID") != -1:
        line = re.sub("^.*ESSID:", "", line)
        line = re.sub('"', '', line)
        line = line.replace('\n', '')
        ssid_list.append(line)
        print(line)

for entry in config.sections():
    if entry != 'global':
        ssid = config.get(entry, 'ssid')
        try:
            i = ssid_list.index(ssid)
        except ValueError:
            continue

        protocol = config.get(entry, 'protocol')

        if 'WEP' == protocol:
            connect_wep(dev, ssid)
        elif 'WPA' == protocol:
            psk = config.get(entry, 'psk')
            print("ssid: " + ssid)
            print("psk: " + psk)
            connect_wpa(dev, ssid, psk)
        else:
            print("ERROR: protocol not recognized")

        net = config.get(entry, 'net')
        if 'dhcp' == net:
            os.system("dhclient " + dev)
        elif 'static' == net:
            address = config.get(entry, 'address')
            gateway = config.get(entry, 'gateway')
            print("address: " + address)
            print("gateway: " + gateway)
            os.system("ifconfig " + dev + " " + address)
            os.system("route add default gw " + gateway + " " + dev)
        try:
            os.system("sh " + config.get(entry, 'post-script'))
        except ConfigParser.NoOptionError:
            break
<|endoftext|>