<commit_before>#!/usr/bin/env python

###############################################################################
#
# This script outputs a CSV table to STDOUT.
# It may use pre-saved YAML file with pillars or get current data by
# querying Salt (if the current host is a Salt minion):
#
#   # Use pre-saved file with pillars.
#   > ./join_hosts_roles_networks.py pillars.yaml > hosts_roles_networks.csv
#   # Query current pillars from Salt.
#   > ./join_hosts_roles_networks.py > hosts_roles_networks.csv
#
# The table is a join of the following objects from pillars:
# * `system_hosts`
# * `system_host_roles`
# * `system_networks`
#
# In other words, the original data from pillars
# is denormalized into CSV table.
#
# The script uses few transformations:
#   *   JSON loaded data to Python object.
#           See:
#               *   http://stackoverflow.com/q/1305532/441652
#   *   The Python objects are use SQLAlchemy Object-Relational Mapping (ORM)
#       to map their data into SQLite DB.
#           Install the following packages first:
#               python-sqlalchemy
#   *   The database is then queried to join all three tables.
#
###############################################################################

import os
import csv
import sys
import yaml
import pprint
import sqlite3
import logging
import datetime
import sqlalchemy

from sqlalchemy.ext.declarative import declarative_base

################################################################################
# Import non-standard modules.

# Determine path for additional modules to import.
start_path = sys.argv[0]
script_dir = os.path.dirname(start_path)
run_dir = os.getcwd()
if os.path.isabs(script_dir):
    modules_dir = os.path.join(
        script_dir,
        'modules',
    )
else:
    modules_dir = os.path.join(
        run_dir,
        script_dir,
        'modules',
    )
sys.path.append(
    modules_dir,
)

import utils.set_log
from utils.exec_command import call_subprocess

################################################################################
# Some globals.

# In-memory SQLite database.
engine = sqlalchemy.create_engine('sqlite://')

################################################################################
# Model for ORM mapping.

Base = declarative_base()

class host_role_to_host_class(Base):

    __tablename__ = 'host_roles_to_hosts'

    host_role_id = sqlalchemy.Column(sqlalchemy.String, sqlalchemy.ForeignKey('host_roles.id'))
    host_id = sqlalchemy.Column(sqlalchemy.String, sqlalchemy.ForeignKey('hosts.id'))

    __table_args__ = (
        sqlalchemy.PrimaryKeyConstraint('host_role_id', 'host_id'),
        {},
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

class network_to_host_class(Base):

    __tablename__ = 'networks_to_hosts'

    network_id = sqlalchemy.Column(sqlalchemy.String, sqlalchemy.ForeignKey('networks.id'))
    host_id = sqlalchemy.Column(sqlalchemy.String, sqlalchemy.ForeignKey('hosts.id'))

    __table_args__ = (
        sqlalchemy.PrimaryKeyConstraint('network_id', 'host_id'),
        {},
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

class host_class(Base):

    __tablename__ = 'hosts'

    id = sqlalchemy.Column(sqlalchemy.String, primary_key = True)
    hostname = sqlalchemy.Column(sqlalchemy.String)
    consider_online_for_remote_connections = sqlalchemy.Column(sqlalchemy.Boolean())

    host_roles_relationship = sqlalchemy.orm.relationship(
        'host_role_class',
        secondary = 'host_roles_to_hosts',
        back_populates = 'hosts_relationship',
    )

    host_networks_relationship = sqlalchemy.orm.relationship(
        'network_class',
        secondary = 'networks_to_hosts',
        back_populates = 'hosts_relationship',
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

class host_role_class(Base):

    __tablename__ = 'host_roles'

    id = sqlalchemy.Column(sqlalchemy.String, primary_key = True)
    hostname = sqlalchemy.Column(sqlalchemy.String)

    hosts_relationship = sqlalchemy.orm.relationship(
        'host_class',
        secondary = 'host_roles_to_hosts',
        back_populates = 'host_roles_relationship',
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

class network_class(Base):

    __tablename__ = 'networks'

    id = sqlalchemy.Column(sqlalchemy.String, primary_key = True)
    broadcast = sqlalchemy.Column(sqlalchemy.String)
    gateway = sqlalchemy.Column(sqlalchemy.String)
    netmask = sqlalchemy.Column(sqlalchemy.String)
    netprefix = sqlalchemy.Column(sqlalchemy.String)
    subnet = sqlalchemy.Column(sqlalchemy.String)

    hosts_relationship = sqlalchemy.orm.relationship(
        'host_class',
        secondary = 'networks_to_hosts',
        back_populates = 'host_networks_relationship',
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

################################################################################
#

def load_database(pillars):

    Base.metadata.create_all(engine)

    DBSession = sqlalchemy.orm.sessionmaker(bind = engine)
    session = DBSession()

    for host_id in pillars['system_hosts'].keys():
        host_d = pillars['system_hosts'][host_id]
        host_d['id'] = host_id
        host_o = host_class(**host_d)
        session.add(host_o)
        for network_id in host_d['host_networks']:
            network_to_host_d = {}
            network_to_host_d['network_id'] = network_id
            network_to_host_d['host_id'] = host_id
            network_to_host_o = network_to_host_class(**network_to_host_d)
            session.add(network_to_host_o)

    for host_role_id in pillars['system_host_roles'].keys():
        host_role_d = pillars['system_host_roles'][host_role_id]
        host_role_d['id'] = host_role_id
        host_role_o = host_role_class(**host_role_d)
        session.add(host_role_o)
        for host_id in host_role_d['assigned_hosts']:
            host_role_to_host_d = {}
            host_role_to_host_d['host_role_id'] = host_role_id
            host_role_to_host_d['host_id'] = host_id
            host_role_to_host_o = host_role_to_host_class(**host_role_to_host_d)
            session.add(host_role_to_host_o)

    for network_id in pillars['system_networks'].keys():
        network_d = pillars['system_networks'][network_id]
        network_d['id'] = network_id
        network_o = network_class(**network_d)
        session.add(network_o)

    session.commit()

    return session

################################################################################
#

def query_database(session):

    a_host_class = sqlalchemy.orm.aliased(host_class)
    b_host_class = sqlalchemy.orm.aliased(host_class)

    query = (
        session.query(
            host_class,
            host_role_class,
            network_class,
        )
        .join(a_host_class, host_role_class.hosts_relationship)
        .join(b_host_class, network_class.hosts_relationship)
    )

    return query

################################################################################
#

def write_query_to_csv(query):

    writer = csv.writer(sys.stdout)

    # Get list of columns for header.
    column_names = []
    for desc in query.column_descriptions:
        column_names += [
            desc['type'].__table__.name + '.' + c
            for c in desc['type'].__table__.columns.keys()
        ]

    # Write header to CSV file.
    writer.writerow(column_names)

    # Write all rows to CSV file.
    for row in query:
        values = []
        for item in row:
            for col in item.__table__.columns:
                values.append(item.__dict__[col.name])

        writer.writerow(values)

################################################################################
#

def load_pillars(
    file_path = None,
):

    salt_output = None

    if file_path is None:

        # Run Salt's `pillar.items` and captrue its output.
        command_args = [
            'sudo',
            'salt-call',
            '--out=yaml',
            'pillar.items',
        ]
        process_output = call_subprocess(
            command_args,
            capture_stdout = True,
        )

        # Parse profile pillars content.
        salt_output = process_output['stdout']

    else:

        with open(file_path, 'r') as data_file:
            salt_output = data_file.read()

    pillars = yaml.load(salt_output)

    # Return profile pillars loaded in memory.
    # NOTE: Output of `salt-call` contain first-level key `local`.
    return pillars['local']

################################################################################
#

def print_yaml(data):

    yaml.dump(
        data,
        sys.stdout,
        default_flow_style = False,
        indent = 4,
    )

################################################################################
#

def main():

    # Set log level.
    utils.set_log.setLoggingLevel('debug')

    # Debug SQL.
    logging.basicConfig()
    logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

    pillars = None
    if len(sys.argv) >= 2:
        pillars = load_pillars(sys.argv[1])
    else:
        pillars = load_pillars()

    session = load_database(pillars)
    query = query_database(session)
    write_query_to_csv(query)
    session.close()

###############################################################################
# MAIN

# Execute further only if this file is executed as a script
# (not imported as a module).
if __name__ == '__main__':
    main()

###############################################################################
# EOF
###############################################################################

<commit_msg>Fix joined table, fix name convention<commit_after>#!/usr/bin/env python

###############################################################################
#
# This script outputs a CSV table to STDOUT.
# It may use pre-saved YAML file with pillars or get current data by
# querying Salt (if the current host is a Salt minion):
#
#   # Use pre-saved file with pillars.
#   > ./join_hosts_roles_networks.py pillars.yaml > hosts_roles_networks.csv
#   # Query current pillars from Salt.
#   > ./join_hosts_roles_networks.py > hosts_roles_networks.csv
#
# The table is a join of the following objects from pillars:
# * `system_hosts`
# * `system_host_roles`
# * `system_networks`
#
# In other words, the original data from pillars
# is denormalized into CSV table.
#
# The script uses few transformations:
#   *   JSON loaded data to Python object.
#           See:
#               *   http://stackoverflow.com/q/1305532/441652
#   *   The Python objects are use SQLAlchemy Object-Relational Mapping (ORM)
#       to map their data into SQLite DB.
#           Install the following packages first:
#               python-sqlalchemy
#   *   The database is then queried to join all three tables.
#
###############################################################################

import os
import csv
import sys
import yaml
import pprint
import sqlite3
import logging
import datetime
import sqlalchemy

from sqlalchemy.ext.declarative import declarative_base

################################################################################
# Import non-standard modules.

# Determine path for additional modules to import.
start_path = sys.argv[0]
script_dir = os.path.dirname(start_path)
run_dir = os.getcwd()
if os.path.isabs(script_dir):
    modules_dir = os.path.join(
        script_dir,
        'modules',
    )
else:
    modules_dir = os.path.join(
        run_dir,
        script_dir,
        'modules',
    )
sys.path.append(
    modules_dir,
)

import utils.set_log
from utils.exec_command import call_subprocess

################################################################################
# Some globals.

# In-memory SQLite database.
engine = sqlalchemy.create_engine('sqlite://')

################################################################################
# Model for ORM mapping.

Base = declarative_base()

class host_roles_to_hosts_class(Base):

    __tablename__ = 'host_roles_to_hosts_table'

    host_role_id = sqlalchemy.Column(sqlalchemy.String, sqlalchemy.ForeignKey('host_roles_table.id'))
    host_id = sqlalchemy.Column(sqlalchemy.String, sqlalchemy.ForeignKey('hosts_table.id'))

    __table_args__ = (
        sqlalchemy.PrimaryKeyConstraint('host_role_id', 'host_id'),
        {},
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

class networks_to_hosts_class(Base):

    __tablename__ = 'networks_to_hosts_table'

    network_id = sqlalchemy.Column(sqlalchemy.String, sqlalchemy.ForeignKey('networks_table.id'))
    host_id = sqlalchemy.Column(sqlalchemy.String, sqlalchemy.ForeignKey('hosts_table.id'))

    __table_args__ = (
        sqlalchemy.PrimaryKeyConstraint('network_id', 'host_id'),
        {},
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

class hosts_class(Base):

    __tablename__ = 'hosts_table'

    id = sqlalchemy.Column(sqlalchemy.String, primary_key = True)
    hostname = sqlalchemy.Column(sqlalchemy.String)
    consider_online_for_remote_connections = sqlalchemy.Column(sqlalchemy.Boolean())

    host_roles_relationship = sqlalchemy.orm.relationship(
        'host_roles_class',
        secondary = 'host_roles_to_hosts_table',
        back_populates = 'hosts_relationship',
    )

    hosts_networks_relationship = sqlalchemy.orm.relationship(
        'networks_class',
        secondary = 'networks_to_hosts_table',
        back_populates = 'hosts_relationship',
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

class host_roles_class(Base):

    __tablename__ = 'host_roles_table'

    id = sqlalchemy.Column(sqlalchemy.String, primary_key = True)
    hostname = sqlalchemy.Column(sqlalchemy.String)

    hosts_relationship = sqlalchemy.orm.relationship(
        'hosts_class',
        secondary = 'host_roles_to_hosts_table',
        #back_populates = 'host_roles_relationship',
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

class networks_class(Base):

    __tablename__ = 'networks_table'

    id = sqlalchemy.Column(sqlalchemy.String, primary_key = True)
    broadcast = sqlalchemy.Column(sqlalchemy.String)
    gateway = sqlalchemy.Column(sqlalchemy.String)
    netmask = sqlalchemy.Column(sqlalchemy.String)
    netprefix = sqlalchemy.Column(sqlalchemy.String)
    subnet = sqlalchemy.Column(sqlalchemy.String)

    hosts_relationship = sqlalchemy.orm.relationship(
        'hosts_class',
        secondary = 'networks_to_hosts_table',
        back_populates = 'hosts_networks_relationship',
    )

    # See: http://stackoverflow.com/a/38929089/441652
    def __init__(self, **entries):
        self.__dict__.update(entries)

################################################################################
#

def load_database(pillars):

    Base.metadata.create_all(engine)

    DBSession = sqlalchemy.orm.sessionmaker(bind = engine)
    session = DBSession()

    for host_id in pillars['system_hosts'].keys():
        logging.info('host_id: ' + host_id)
        host_d = pillars['system_hosts'][host_id]
        host_d['id'] = host_id
        host_o = hosts_class(**host_d)
        session.add(host_o)
        for network_id in host_d['host_networks']:
            logging.info('host_id:network_id: ' + host_id + ':' + network_id)
            network_to_host_d = {}
            network_to_host_d['network_id'] = network_id
            network_to_host_d['host_id'] = host_id
            network_to_host_o = networks_to_hosts_class(**network_to_host_d)
            session.add(network_to_host_o)

    for host_role_id in pillars['system_host_roles'].keys():
        logging.info('host_role_id: ' + host_role_id)
        host_role_d = pillars['system_host_roles'][host_role_id]
        host_role_d['id'] = host_role_id
        host_role_o = host_roles_class(**host_role_d)
        session.add(host_role_o)
        for host_id in host_role_d['assigned_hosts']:
            logging.info('host_role_id:host_id: ' + host_role_id + ':' + host_id)
            host_role_to_host_d = {}
            host_role_to_host_d['host_role_id'] = host_role_id
            host_role_to_host_d['host_id'] = host_id
            host_role_to_host_o = host_roles_to_hosts_class(**host_role_to_host_d)
            session.add(host_role_to_host_o)

    for network_id in pillars['system_networks'].keys():
        logging.info('network_id: ' + network_id)
        network_d = pillars['system_networks'][network_id]
        network_d['id'] = network_id
        network_o = networks_class(**network_d)
        session.add(network_o)

    session.commit()

    return session

################################################################################
#

def query_database(session):

    # Because the same table is used in different context,
    # we are forced to use aliases.
    a_hosts_class = sqlalchemy.orm.aliased(hosts_class)
    b_hosts_class = sqlalchemy.orm.aliased(hosts_class)

    query = (
        session.query(
            a_hosts_class,
            b_hosts_class,
            host_roles_class,
            networks_class,
        )
        .join(a_hosts_class, host_roles_class.hosts_relationship)
        .join(b_hosts_class, networks_class.hosts_relationship)
        # Not sure how to join `a_hosts_class` and `b_hosts_class` -
        # there are conflicting columns ids - `filter` does the job.
        .filter(a_hosts_class.id == b_hosts_class.id)
        .with_entities(
            # Reduce number of identical columns by selecting only
            # one aliased entity.
            a_hosts_class,
            host_roles_class,
            networks_class,
        )
    )

    return query

################################################################################
#

def write_query_to_csv(query):

    writer = csv.writer(sys.stdout)

    # Get list of columns for header.
    column_names = []
    for desc in query.column_descriptions:
        column_names += [
            desc['type'].__table__.name + '.' + c
            for c in desc['type'].__table__.columns.keys()
        ]

    # Write header to CSV file.
    writer.writerow(column_names)

    # Write all rows to CSV file.
    for row in query:
        values = []
        for item in row:
            for col in item.__table__.columns:
                values.append(item.__dict__[col.name])

        writer.writerow(values)

################################################################################
#

def load_pillars(
    file_path = None,
):

    salt_output = None

    if file_path is None:

        logging.info("File path with pillars is not specified - getting pillars from Salt.")

        # Run Salt's `pillar.items` and captrue its output.
        command_args = [
            'sudo',
            'salt-call',
            '--out=yaml',
            'pillar.items',
        ]
        process_output = call_subprocess(
            command_args,
            capture_stdout = True,
        )

        # Parse profile pillars content.
        salt_output = process_output['stdout']

    else:

        with open(file_path, 'r') as data_file:
            salt_output = data_file.read()

    pillars = yaml.load(salt_output)

    # Return profile pillars loaded in memory.
    # NOTE: Output of `salt-call` contain first-level key `local`.
    return pillars['local']

################################################################################
#

def print_yaml(data):

    yaml.dump(
        data,
        sys.stdout,
        default_flow_style = False,
        indent = 4,
    )

################################################################################
#

def main():

    # Set log level.
    utils.set_log.setLoggingLevel('debug')

    # Debug SQL.
    logging.basicConfig()
    logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

    pillars = None
    if len(sys.argv) >= 2:
        pillars = load_pillars(sys.argv[1])
    else:
        pillars = load_pillars()

    session = load_database(pillars)
    query = query_database(session)
    write_query_to_csv(query)
    session.close()

###############################################################################
# MAIN

# Execute further only if this file is executed as a script
# (not imported as a module).
if __name__ == '__main__':
    main()

###############################################################################
# EOF
###############################################################################

<|endoftext|>


<commit_before><commit_msg>asfpy: VolService<commit_after><|endoftext|>


<commit_before>#!/usr/local/bin/python2.7
import fileinput
import glob
import os
import re
import sys
import datetime

class JackAnalyzer:
    def __init__(self, source, dest):
        self.dest = dest
        self.tokenizer = JackTokenizer(source, dest)
        self.compilationEngine = CompilationEngine(self.tokenizer, dest)
        return

class JackTokenizer:
    keywords = 'class', 'constructor', 'function', 'method', 'field', \
               'static', 'var', 'int', 'char', 'boolean', 'void', 'true', \
               'false', 'null', 'this', 'let', 'do', 'if', 'else', 'while', \
               'return'
    symbols = '{', '}', '(', ')', '[', ']', '.', ',', ';', '+', '-', '*', \
              '/', '&', ',', '|', '<', '>', '=', '~'
    # split() note: "f capturing parentheses are used in pattern, then the text of all groups
    # in the pattern are also returned as part of the resulting list". We save the symbols
    # but purposefully lose the whitespace.
    reSymbols = re.compile(r'([{\}()\[\],.;+\-*/&|<>=~])|\s+')
    reIdentifier = re.compile(r'^[a-zA-Z]{1}[a-zA-Z0-9]*$')
    reIntegerConstant = re.compile(r'^\d+$')

    def __init__(self, source, dest):
        self.source = source
        self.dest = dest
        self.token = None
        self.tokens = self.readAllTokens()
        return

    def xml(self):
        self.dest.write('<tokens>\n')
        for token in self.tokens:
            if token.type == Token.KEYWORD:
                self.dest.write('<keyword> ' + token.keyWord + ' </keyword>')
            elif token.type == Token.SYMBOL:
                self.dest.write('<symbol> ' + escapeSymbol(token.symbol) + ' </symbol>')
            elif token.type == Token.IDENTIFIER:
                self.dest.write('<identifier> ' + token.identifier + ' </identifier>')
            elif token.type == Token.INT_CONST:
                self.dest.write('<integerConstant> ' + str(token.intVal) + ' </integerConstant>')
            elif token.type == Token.STRING_CONST:
                self.dest.write('<stringConstant> ' + token.stringVal + ' </stringConstant>')
            else:
                sys.stderr.write("I know not this token: \"" + token.type + "\"\n")
                sys.exit(3)
            self.dest.write('\n')
        self.dest.write('</tokens>\n')

    def hasMoreTokens(self):
        if len(self.tokens) > 0:
            return True
        else:
            return False

    def advance(self):
        self.token = self.tokens.pop(0)

    def tokenType(self):
        return self.token.type

    def symbol(self):
        return self.token.symbol

    def identifier(self):
        return self.token.identifier

    def intVal(self):
        return self.token.intVal

    def stringVal(self):
        return self.token.stringVal

    def readAllTokens(self):
        tokens = []
        inputStream = self.source.read()
        inputStream = re.sub(r'//.*$', '', inputStream, flags=re.MULTILINE)  # strip comments `//`
        inputStream = re.sub(r'/\*.*?\*/', '', inputStream, flags=re.MULTILINE | re.DOTALL)  # strip comments `/* */`
        while inputStream:
            inputStream = inputStream.lstrip()  # strip leading whitespace, if any
            if re.match(r'"', inputStream):
                _, field, inputStream = re.split(r'"', inputStream, 2)
                tokens.append(Token(Token.STRING_CONST, stringVal=field))
            else:
                # Hack alert: the split below will sometimes inject None and '' into the array
                # leaving it with 3 elements & throwing it off. We filter to remove those.
                # e.g. reSymbolsAndWhitespace.split('x {', 1) = ['x', None, '{']
                # matches = [x for x in JackTokenizer.reSymbols.split(inputStream, 1) if x is not None]
                matches = list(filter(None, JackTokenizer.reSymbols.split(inputStream, 1)))
                if len(matches) == 3:
                    field = matches[0]
                    inputStream = matches[1] + matches[2]
                elif len(matches) == 2:
                    field, inputStream = matches
                elif len(matches) == 0:
                    break
                else:
                    sys.stderr.write(str(len(matches)) + ' MATCHES ' + str(matches[:-1]) + '\n')
                    sys.stderr.write("input: " + inputStream + '\n')
                # Lame coding alert: I shouldn't have empty strings
                # and I shouldn't skip them by using `pass`. This
                # code is double-lame, but I'll never fix it, sorry.
                if field in JackTokenizer.keywords:
                    tokens.append(Token(Token.KEYWORD, keyWord=field))
                elif field in JackTokenizer.symbols:
                    tokens.append(Token(Token.SYMBOL, symbol=field))
                elif JackTokenizer.reIdentifier.match(field):
                    tokens.append(Token(Token.IDENTIFIER, identifier=field))
                elif JackTokenizer.reIntegerConstant.match(field):
                    tokens.append(Token(Token.INT_CONST, intVal=int(field)))
                else:
                    sys.stderr.write("I can't figure out this field: \"" + field + "\"!")
                    sys.exit(1)

        return (tokens)

# Ensure XML-dangerous symbols are properly escaped
def escapeSymbol(symbol):
    return {
        '<': '<',
        '>': '>',
        '"': '"',
        '&': '&',
    }.get(symbol, symbol)

class Token:
    def __init__(self, type, keyWord=None, symbol=None, identifier=None, intVal=None, stringVal=None):
        self.type = type
        self.keyWord = keyWord
        self.symbol = symbol
        self.symbol = symbol
        self.identifier = identifier
        self.intVal = intVal
        self.stringVal = stringVal

    # Token Types
    KEYWORD = 1
    SYMBOL = 2
    IDENTIFIER = 3
    INT_CONST = 4
    STRING_CONST = 5

    # Key Words (for Token Type KEYWORD)
    CLASS = 10
    METHOD = 11
    FUNCTION = 12
    CONSTRUCTOR = 13
    INT = 14
    BOOLEAN = 15
    CHAR = 16
    VOID = 17
    VAR = 18
    STATIC = 19
    FIELD = 20
    LET = 21
    DO = 22
    IF = 23
    ELSE = 24
    WHILE = 25
    RETURN = 26
    TRUE = 27
    FALSE = 28
    NULL = 29
    THIS = 30

class CompilationEngine:
    def __init__(self, tokenizer, dest):
        self.dest = dest
        self.tokenizer = tokenizer

    def CompileClass(self):
        pass

    def CompileClassVarDec(self):
        pass

    def CompileSubroutine(self):
        pass

    def compileParameterList(self):
        pass

    def compileVarDec(self):
        pass

    def compileStatements(self):
        pass

    def compileDo(self):
        pass

    def compileLet(self):
        pass

    def compileWhile(self):
        pass

    def compileReturn(self):
        pass

    def compileIf(self):
        pass

    def CompileExpression(self):
        pass

    def CompileTerm(self):
        pass

    def CompileExpressionList(self):
        pass

    def xml(self):
        pass

# Open the file & Analyze

# return a list of filenames to compile
def jackSourceFilenames():
    cmd_name = sys.argv[0].split('/')[-1]
    sourceFilenames = glob.glob('*.jack')  # initialize to all the .jack files in PWD
    sys.stderr.write('sys.argv: ' + str(sys.argv) + '\n')

    if len(sys.argv) > 2:
        sys.exit(cmd_name + " error: pass me one arg, the name of the file to compile, or a directory, or no argument")

    if len(sys.argv) == 2:
        file_or_directory = sys.argv[1]
        if os.path.isfile(file_or_directory):
            sourceFilenames = [file_or_directory]
        elif os.path.isdir(file_or_directory):
            directory = file_or_directory.rstrip('/')
            sourceFilenames = glob.glob(os.path.join(directory, '*.jack'))
        else:
            sys.exit(cmd_name + " error: " + file_or_directory + " isn't a file or directory!")
    return sourceFilenames

# main
cmd_name = sys.argv[0].split('/')[-1]
for sourceFileName in jackSourceFilenames():
    try:
        source = open(sourceFileName, "r")
    except:
        sys.exit(cmd_name + " error. I couldn't open " + str(sourceFileName) + " for reading!")
    sys.stderr.write(cmd_name + ": opened " + sourceFileName + " for reading.\n")
    destFileName = sourceFileName.replace('.jack', 'TT.xml')
    try:
        dest = open(destFileName, "w")
    except:
        sys.exit(cmd_name + " error. I couldn't open " + str(sourceFileName) + " for reading!")
    sys.stderr.write(cmd_name + ": opened " + destFileName + " for writing.\n")

    analyzer = JackAnalyzer(source, dest)
    analyzer.tokenizer.xml()
<commit_msg>Compile Class works!<commit_after>#!/usr/local/bin/python2.7
import glob
import os
import re
import sys

class JackAnalyzer:
    def __init__(self, source, dest):
        self.dest = dest
        self.tokenizer = JackTokenizer(source, dest)
        self.compilationEngine = CompilationEngine(self.tokenizer, dest)
        return

class JackTokenizer:
    keywords = 'class', 'constructor', 'function', 'method', 'field', \
               'static', 'var', 'int', 'char', 'boolean', 'void', 'true', \
               'false', 'null', 'this', 'let', 'do', 'if', 'else', 'while', \
               'return'
    symbols = '{', '}', '(', ')', '[', ']', '.', ',', ';', '+', '-', '*', \
              '/', '&', ',', '|', '<', '>', '=', '~'
    # split() note: "f capturing parentheses are used in pattern, then the text of all groups
    # in the pattern are also returned as part of the resulting list". We save the symbols
    # but purposefully lose the whitespace.
    reSymbols = re.compile(r'([{\}()\[\],.;+\-*/&|<>=~])|\s+')
    reIdentifier = re.compile(r'^[a-zA-Z]{1}[a-zA-Z0-9]*$')
    reIntegerConstant = re.compile(r'^\d+$')

    def __init__(self, source, dest):
        self.source = source
        self.dest = dest
        self.token = None
        self.tokens = self.readAllTokens()
        return

    def xml(self):
        self.dest.write('<tokens>\n')
        for token in self.tokens:
            if token.type == Token.KEYWORD:
                self.dest.write('<keyword> ' + token.keyWord + ' </keyword>')
            elif token.type == Token.SYMBOL:
                self.dest.write('<symbol> ' + escapeSymbol(token.symbol) + ' </symbol>')
            elif token.type == Token.IDENTIFIER:
                self.dest.write('<identifier> ' + token.identifier + ' </identifier>')
            elif token.type == Token.INT_CONST:
                self.dest.write('<integerConstant> ' + str(token.intVal) + ' </integerConstant>')
            elif token.type == Token.STRING_CONST:
                self.dest.write('<stringConstant> ' + token.stringVal + ' </stringConstant>')
            else:
                sys.stderr.write("I know not this token: \"" + token.type + "\"\n")
                sys.exit(3)
            self.dest.write('\n')
        self.dest.write('</tokens>\n')

    def hasMoreTokens(self):
        if len(self.tokens) > 0:
            return True
        else:
            return False

    def advance(self):
        self.token = self.tokens.pop(0)

    def tokenType(self):
        return self.token.type

    def symbol(self):
        return self.token.symbol

    def identifier(self):
        return self.token.identifier

    def intVal(self):
        return self.token.intVal

    def stringVal(self):
        return self.token.stringVal

    def readAllTokens(self):
        tokens = []
        inputStream = self.source.read()
        inputStream = re.sub(r'//.*$', '', inputStream, flags=re.MULTILINE)  # strip comments `//`
        inputStream = re.sub(r'/\*.*?\*/', '', inputStream, flags=re.MULTILINE | re.DOTALL)  # strip comments `/* */`
        while inputStream:
            inputStream = inputStream.lstrip()  # strip leading whitespace, if any
            if re.match(r'"', inputStream):
                _, field, inputStream = re.split(r'"', inputStream, 2)
                tokens.append(Token(Token.STRING_CONST, stringVal=field))
            else:
                # Hack alert: the split below will sometimes inject None and '' into the array
                # leaving it with 3 elements & throwing it off. We filter to remove those.
                # e.g. reSymbolsAndWhitespace.split('x {', 1) = ['x', None, '{']
                # matches = [x for x in JackTokenizer.reSymbols.split(inputStream, 1) if x is not None]
                matches = list(filter(None, JackTokenizer.reSymbols.split(inputStream, 1)))
                if len(matches) == 3:
                    field = matches[0]
                    inputStream = matches[1] + matches[2]
                elif len(matches) == 2:
                    field, inputStream = matches
                elif len(matches) == 0:
                    break
                else:
                    sys.stderr.write(str(len(matches)) + ' MATCHES ' + str(matches[:-1]) + '\n')
                    sys.stderr.write("input: " + inputStream + '\n')
                # Lame coding alert: I shouldn't have empty strings
                # and I shouldn't skip them by using `pass`. This
                # code is double-lame, but I'll never fix it, sorry.
                if field in JackTokenizer.keywords:
                    tokens.append(Token(Token.KEYWORD, keyWord=field))
                elif field in JackTokenizer.symbols:
                    tokens.append(Token(Token.SYMBOL, symbol=field))
                elif JackTokenizer.reIdentifier.match(field):
                    tokens.append(Token(Token.IDENTIFIER, identifier=field))
                elif JackTokenizer.reIntegerConstant.match(field):
                    tokens.append(Token(Token.INT_CONST, intVal=int(field)))
                else:
                    sys.stderr.write("I can't figure out this field: \"" + field + "\"!")
                    sys.exit(1)

        return (tokens)

# Ensure XML-dangerous symbols are properly escaped
def escapeSymbol(symbol):
    return {
        '<': '<',
        '>': '>',
        '"': '"',
        '&': '&',
    }.get(symbol, symbol)

class Token:
    def __init__(self, type, keyWord=None, symbol=None, identifier=None, intVal=None, stringVal=None):
        self.type = type
        self.keyWord = keyWord
        self.symbol = symbol
        self.identifier = identifier
        self.intVal = intVal
        self.stringVal = stringVal

    def __repr__(self):
        if self is None:
            return "Token{None}"
        elif self.type == Token.KEYWORD:
            return "Token{keyword: " + self.keyWord + "}"
        elif self.type == Token.SYMBOL:
            return "Token(symbol: \"" + self.symbol + "\"}"
        elif self.type == Token.IDENTIFIER:
            return "Token{identifier: " + self.identifier + "}"
        elif self.type == Token.INT_CONST:
            return "Token{int_const: " + self.intVal + "}"
        elif self.type == Token.STRING_CONST:
            return "Token{string_const: \"" + self.stringVal + "\"}"
        else:
            return "Token{weird: strange}"

    # Token Types
    KEYWORD = 1
    SYMBOL = 2
    IDENTIFIER = 3
    INT_CONST = 4
    STRING_CONST = 5

    # Key Words (for Token Type KEYWORD)
    CLASS = "class"
    METHOD = "method"
    FUNCTION = "function"
    CONSTRUCTOR = "constructor"
    INT = "int"
    BOOLEAN = "boolean"
    CHAR = "char"
    VOID = "void"
    VAR = "var"
    STATIC = "static"
    FIELD = "field"
    LET = "let"
    DO = "do"
    IF = "if"
    ELSE = "else"
    WHILE = "while"
    RETURN = "return"
    TRUE = "true"
    FALSE = "false"
    NULL = "null"
    THIS = "this"

class CompilationEngine:

    def __init__(self, tokenizer, dest):
        self.dest = dest
        self.tokenizer = tokenizer
        token = self.tokenizer.token
        if token != None:
            unexpectedToken(token)
        self.indent = ''
        while (self.tokenizer.hasMoreTokens()):
            self.tokenizer.advance()
            token = self.tokenizer.token
            if token.type == Token.KEYWORD and token.keyWord == Token.CLASS:
                self.CompileClass()
            else:
                unexpectedToken(token)

    def CompileClass(self):
        self.dest.write(self.indent)
        self.dest.write("<class>\n")
        original_indent = self.indent
        self.indent += '  '
        self.dest.write(self.indent)
        self.dest.write("<keyword> class </keyword>\n")
        self.tokenizer.advance()
        token = self.tokenizer.token
        if token.type != Token.IDENTIFIER:
            unexpectedToken(token)
        self.dest.write(self.indent)
        self.dest.write("<identifier> " + token.identifier + " </identifier>\n")
        self.tokenizer.advance()
        token = self.tokenizer.token
        if token.type != Token.SYMBOL or token.symbol != '{':
            unexpectedToken(token)
        self.dest.write(self.indent)
        self.dest.write("<symbol> " + token.symbol + " </symbol>\n")
        self.CompileClassVarDec()
        self.indent = original_indent

    def CompileClassVarDec(self):
        pass

    def CompileSubroutine(self):
        pass

    def compileParameterList(self):
        pass

    def compileVarDec(self):
        pass

    def compileStatements(self):
        pass

    def compileDo(self):
        pass

    def compileLet(self):
        pass

    def compileWhile(self):
        pass

    def compileReturn(self):
        pass

    def compileIf(self):
        pass

    def CompileExpression(self):
        pass

    def CompileTerm(self):
        pass

    def CompileExpressionList(self):
        pass

    def xml(self):
        pass

def unexpectedToken(token):
    sys.stderr.write("Unexpected token: " + str(token) + "!\n")
    sys.stderr.write(token)  # force a stacktrace
    sys.exit(5)

# return a list of filenames to compile
def jackSourceFilenames():
    cmd_name = sys.argv[0].split('/')[-1]
    sourceFilenames = glob.glob('*.jack')  # initialize to all the .jack files in PWD
    sys.stderr.write('sys.argv: ' + str(sys.argv) + '\n')

    if len(sys.argv) > 2:
        sys.exit(cmd_name + " error: pass me one arg, the name of the file to compile, or a directory, or no argument")

    if len(sys.argv) == 2:
        file_or_directory = sys.argv[1]
        if os.path.isfile(file_or_directory):
            sourceFilenames = [file_or_directory]
        elif os.path.isdir(file_or_directory):
            directory = file_or_directory.rstrip('/')
            sourceFilenames = glob.glob(os.path.join(directory, '*.jack'))
        else:
            sys.exit(cmd_name + " error: " + file_or_directory + " isn't a file or directory!")
    return sourceFilenames

# main
cmd_name = sys.argv[0].split('/')[-1]
for sourceFileName in jackSourceFilenames():
    try:
        source = open(sourceFileName, "r")
    except:
        sys.exit(cmd_name + " error. I couldn't open " + str(sourceFileName) + " for reading!")
    sys.stderr.write(cmd_name + ": opened " + sourceFileName + " for reading.\n")
    destFileName = sourceFileName.replace('.jack', 'X.xml')
    try:
        dest = open(destFileName, "w")
    except:
        sys.exit(cmd_name + " error. I couldn't open " + str(sourceFileName) + " for reading!")
    sys.stderr.write(cmd_name + ": opened " + destFileName + " for writing.\n")

    analyzer = JackAnalyzer(source, dest)
    analyzer.tokenizer.xml()
<|endoftext|>


<commit_before>from flask import render_template, redirect, url_for, abort, flash, request,\
    current_app
from flask.ext.login import login_required, current_user
from . import main
from .forms import EditProfileForm, EditProfileAdminForm, PostForm
from .. import db
from ..models import Permission, Role, User, Post
from ..decorators import admin_required, permission_required

@main.route('/', methods=['GET', 'POST'])
def index():
    form = PostForm()
    if current_user.can(Permission.WRITE_ARTICLES) and \
            form.validate_on_submit():
        post = Post(body=form.body.data,
                    author=current_user._get_current_object())
        db.session.add(post)
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = Post.query.order_by(Post.timestamp.desc()).paginate(
        page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'],
        error_out=False)
    posts = pagination.items
    return render_template('index.html', form=form, posts=posts,
                           pagination=pagination)

@main.route('/user/<username>')
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    page = request.args.get('page', 1, type=int)
    pagination = user.posts.order_by(Post.timestamp.desc()).paginate(
        page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'],
        error_out=False)
    posts = pagination.items
    return render_template('user.html', user=user, posts=posts,
                           pagination=pagination)

@main.route('/edit-profile', methods=['GET', 'POST'])
@login_required
def edit_profile():
    form = EditProfileForm()
    if form.validate_on_submit():
        current_user.name = form.name.data
        current_user.location = form.location.data
        current_user.about_me = form.about_me.data
        db.session.add(current_user)
        flash('Your profile has been updated.')
        return redirect(url_for('.user', username=current_user.username))
    form.name.data = current_user.name
    form.location.data = current_user.location
    form.about_me.data = current_user.about_me
    return render_template('edit_profile.html', form=form)

@main.route('/edit-profile/<int:id>', methods=['GET', 'POST'])
@login_required
@admin_required
def edit_profile_admin(id):
    user = User.query.get_or_404(id)
    form = EditProfileAdminForm(user=user)
    if form.validate_on_submit():
        user.email = form.email.data
        user.username = form.username.data
        user.confirmed = form.confirmed.data
        user.role = Role.query.get(form.role.data)
        user.name = form.name.data
        user.location = form.location.data
        user.about_me = form.about_me.data
        db.session.add(user)
        flash('The profile has been updated.')
        return redirect(url_for('.user', username=user.username))
    form.email.data = user.email
    form.username.data = user.username
    form.confirmed.data = user.confirmed
    form.role.data = user.role_id
    form.name.data = user.name
    form.location.data = user.location
    form.about_me.data = user.about_me
    return render_template('edit_profile.html', form=form, user=user)

@main.route('/post/<int:id>')
def post(id):
    post = Post.query.get_or_404(id)
    return render_template('post.html', posts=[post])

@main.route('/edit/<int:id>', methods=['GET', 'POST'])
@login_required
def edit(id):
    post = Post.query.get_or_404(id)
    if current_user != post.author and \
            not current_user.can(Permission.ADMINISTER):
        abort(403)
    form = PostForm()
    if form.validate_on_submit():
        post.body = form.body.data
        db.session.add(post)
        flash('The post has been updated.')
        return redirect(url_for('.post', id=post.id))
    form.body.data = post.body
    return render_template('edit_post.html', form=form)

@main.route('/follow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def follow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if current_user.is_following(user):
        flash('You are already following this user.')
        return redirect(url_for('.user', username=username))
    current_user.follow(user)
    flash('You are now following %s.' % username)
    return redirect(url_for('.user', username=username))

@main.route('/unfollow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def unfollow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if not current_user.is_following(user):
        flash('You are not following this user.')
        return redirect(url_for('.user', username=username))
    current_user.unfollow(user)
    flash('You are not following %s anymore.' % username)
    return redirect(url_for('.user', username=username))

@main.route('/followers/<username>')
def followers(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followers.paginate(
        page, per_page=current_app.config['FLASKY_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.follower, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followers of",
                           endpoint='.followers', pagination=pagination,
                           follows=follows)

@main.route('/followed-by/<username>')
def followed_by(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followed.paginate(
        page, per_page=current_app.config['FLASKY_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.followed, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followed by",
                           endpoint='.followed_by', pagination=pagination,
                           follows=follows)
<commit_msg>home page followed posts<commit_after>from flask import render_template, redirect, url_for, abort, flash, request,\
    current_app
from flask.ext.login import login_required, current_user
from . import main
from .forms import EditProfileForm, EditProfileAdminForm, PostForm
from .. import db
from ..models import Permission, Role, User, Post
from ..decorators import admin_required, permission_required

@main.route('/', methods=['GET', 'POST'])
def index():
    form = PostForm()
    if current_user.can(Permission.WRITE_ARTICLES) and \
            form.validate_on_submit():
        post = Post(body=form.body.data,
                    author=current_user._get_current_object())
        db.session.add(post)
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = Post.query.order_by(Post.timestamp.desc()).paginate(
        page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'],
        error_out=False)
    posts = pagination.items

    show_followed = False
    if current_user.is_authenticated():
        show_followed = bool(request.cookied.get('show_followed', ' '))
    if show_followed:
        query =current_user.followed_posts
    else:
        query = Post.query
    pagination = query.order_by(Post.timestamp.desc()).pagination(page,
        per_page = current_app.config['FLASKY_POSTS_PER_PAGE'],
        error_out = False)
    posts = pagination.items
    return render_template('index.html', form=form, posts=posts,
                           show_followde = show_followed, pagination=pagination)

@main.route('/user/<username>')
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    page = request.args.get('page', 1, type=int)
    pagination = user.posts.order_by(Post.timestamp.desc()).paginate(
        page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'],
        error_out=False)
    posts = pagination.items
    return render_template('user.html', user=user, posts=posts,
                           pagination=pagination)

@main.route('/edit-profile', methods=['GET', 'POST'])
@login_required
def edit_profile():
    form = EditProfileForm()
    if form.validate_on_submit():
        current_user.name = form.name.data
        current_user.location = form.location.data
        current_user.about_me = form.about_me.data
        db.session.add(current_user)
        flash('Your profile has been updated.')
        return redirect(url_for('.user', username=current_user.username))
    form.name.data = current_user.name
    form.location.data = current_user.location
    form.about_me.data = current_user.about_me
    return render_template('edit_profile.html', form=form)

@main.route('/edit-profile/<int:id>', methods=['GET', 'POST'])
@login_required
@admin_required
def edit_profile_admin(id):
    user = User.query.get_or_404(id)
    form = EditProfileAdminForm(user=user)
    if form.validate_on_submit():
        user.email = form.email.data
        user.username = form.username.data
        user.confirmed = form.confirmed.data
        user.role = Role.query.get(form.role.data)
        user.name = form.name.data
        user.location = form.location.data
        user.about_me = form.about_me.data
        db.session.add(user)
        flash('The profile has been updated.')
        return redirect(url_for('.user', username=user.username))
    form.email.data = user.email
    form.username.data = user.username
    form.confirmed.data = user.confirmed
    form.role.data = user.role_id
    form.name.data = user.name
    form.location.data = user.location
    form.about_me.data = user.about_me
    return render_template('edit_profile.html', form=form, user=user)

@main.route('/post/<int:id>')
def post(id):
    post = Post.query.get_or_404(id)
    return render_template('post.html', posts=[post])

@main.route('/edit/<int:id>', methods=['GET', 'POST'])
@login_required
def edit(id):
    post = Post.query.get_or_404(id)
    if current_user != post.author and \
            not current_user.can(Permission.ADMINISTER):
        abort(403)
    form = PostForm()
    if form.validate_on_submit():
        post.body = form.body.data
        db.session.add(post)
        flash('The post has been updated.')
        return redirect(url_for('.post', id=post.id))
    form.body.data = post.body
    return render_template('edit_post.html', form=form)

@main.route('/follow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def follow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if current_user.is_following(user):
        flash('You are already following this user.')
        return redirect(url_for('.user', username=username))
    current_user.follow(user)
    flash('You are now following %s.' % username)
    return redirect(url_for('.user', username=username))

@main.route('/unfollow/<username>')
@login_required
@permission_required(Permission.FOLLOW)
def unfollow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    if not current_user.is_following(user):
        flash('You are not following this user.')
        return redirect(url_for('.user', username=username))
    current_user.unfollow(user)
    flash('You are not following %s anymore.' % username)
    return redirect(url_for('.user', username=username))

@main.route('/followers/<username>')
def followers(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followers.paginate(
        page, per_page=current_app.config['FLASKY_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.follower, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followers of",
                           endpoint='.followers', pagination=pagination,
                           follows=follows)

@main.route('/followed-by/<username>')
def followed_by(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('Invalid user.')
        return redirect(url_for('.index'))
    page = request.args.get('page', 1, type=int)
    pagination = user.followed.paginate(
        page, per_page=current_app.config['FLASKY_FOLLOWERS_PER_PAGE'],
        error_out=False)
    follows = [{'user': item.followed, 'timestamp': item.timestamp}
               for item in pagination.items]
    return render_template('followers.html', user=user, title="Followed by",
                           endpoint='.followed_by', pagination=pagination,
                           follows=follows)
<|endoftext|>


<commit_before>#!/usr/bin/python
# vim: set fileencoding=UTF-8 :
"""
load

todo doc
"""
import sys,os,getopt
import base64, http.client
import urllib.request
import urllib.error
import ijson.backends.yajl2_cffi as ijson
import json
from time import localtime, strftime

import  dboperator

def show(message):
  print(strftime("%Y-%m-%d %H:%M:%S", localtime())+" "+message)

def load(secure,hostname,url,schema,table,postdata,condition,verbose,rowcount):
  show("begin "+hostname+" "+url+" "+schema+" "+table+" "+(postdata or "")+" "+(condition or ""))
  if secure:
    address = "https://"+hostname+url
  else:
    address = "http://"+hostname+url
  show("load from "+address)

  reqheaders = {'Content-Type': 'application/json'}
  reqheaders['Caller-Id'] = '1.2.246.562.10.2013112012294919827487.vipunen'

  # api credentials from env vars
  if os.getenv("API_USERNAME"):
    show("using authentication")
    apiuser = os.getenv("API_USERNAME")
    apipass = os.getenv("API_PASSWORD")
    reqheaders['Authorization'] = 'Basic %s' % base64.b64encode(apiuser+":"+apipass)

  # automatic POST with (post)data
  print("value used for , -r, --rowcount=", rowcount)
  request = urllib.request.urlopen(address, data=postdata, headers=reqheaders)
  try:
    response = urllib.request.urlopen(request)
  except http.client.IncompleteRead as e:
    show('IncompleteRead exception.')
    show('Received: %d'%(e.partial))
    sys.exit(2)
except urllib.error.HTTPError as e:
    show('The server couldn\'t fulfill the request.')
    show('Error code: %d'%(e.code))
    sys.exit(2)
except urllib.error.URLError as e:
    show('We failed to reach a server.')
    show('Reason: %s'%(e.reason))
    sys.exit(2)
  else:
    # everything is fine
    show("api call OK")

  # remove data conditionally, otherwise empty
  # merge operation could be considered here...
  if condition:
    show("remove from %s.%s with condition '%s'"%(schema,table,condition))
    dboperator.execute("DELETE FROM %s.%s WHERE %s"%(schema,table,condition))
  else:
    show("empty %s.%s"%(schema,table))
    dboperator.empty(schema,table)

  show("insert data")
  cnt=0
  manycount = 0
  rows = []

  for row in ijson.items(response,'item'):
        cnt+=1
        manycount+=1
        # show some sign of being alive
        if cnt%100 == 0:
          sys.stdout.write('.')
          sys.stdout.flush()
        if cnt%1000 == 0:
          show("-- %d" % (cnt))
        if verbose: show("%d -- %s"%(cnt,row))

        # find out which columns to use on insert
        dboperator.resetcolumns(row)

        # flatten arrays/lists
        for col in row:
            if type(row[col]) is list:
                row[col] = ''.join(map(str,json.dumps(row[col])))
        rows.append(row)
        if cnt == 1:
            dboperator.insert(address,schema,table,row)
            manycount = 0
            rows = []
        if cnt > 1:
            if manycount == rowcount:
                insert(address,schema,table,rows)
                manycount = 0
                rows = []
  if len(rows) <= manycount and len(rows) > 0:
      insert(address,schema,table,rows)
      rows = []
      manycount = 0

  show("wrote %d"%(cnt))
  show("ready")

def insert(address,schema,table,rows):
    dboperator.insertMany(address,schema,table,rows)

def usage():
  print("""
usage: load.py [-s|--secure] -H|--hostname <hostname> -u|--url <url> -e|--schema <schema> -t|--table <table> [-p|--postdata] [-c|--condition <condition>] [-v|--verbose [-r]--rowcount]
""")

def main(argv):
  # muuttujat jotka kerrotaan argumentein
  secure=False
  hostname,url,schema,table="","","",""
  postdata=None
  condition=None
  verbose=False
  rowcount = 500

  try:
    opts,args=getopt.getopt(argv,"sH:u:e:t:p:c:v:r",["secure","hostname=","url=","schema=","table=","postdata=","condition=","verbose","rowcount="])
  except getopt.GetoptError as err:
    print(err)
    usage()
    sys.exit(2)
  for opt,arg in opts:
    if opt in ("-s", "--secure"): secure=True
    elif opt in ("-H", "--hostname"): hostname=arg
    elif opt in ("-u", "--url"): url=arg
    elif opt in ("-e", "--schema"): schema=arg
    elif opt in ("-t", "--table"): table=arg
    elif opt in ("-p", "--postdata"): postdata=arg
    elif opt in ("-c", "--condition"): condition=arg
    elif opt in ("-v", "--verbose"): verbose=True
    elif opt in ("-r", "--rowcount"): rowcount=arg
  if not hostname or not url or not schema or not table:
    usage()
    sys.exit(2)

  load(secure,hostname,url,schema,table,postdata,condition,verbose, rowcount)

  dboperator.close()

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>CSCVIPUNEN-1311 load_v2 to python3<commit_after>#!/usr/bin/python
# vim: set fileencoding=UTF-8 :
"""
load

todo doc
"""
import sys,os,getopt
import base64, http.client
import urllib.request
import urllib.error
import ijson.backends.yajl2_cffi as ijson
import json
from time import localtime, strftime

import  dboperator

def show(message):
  print(strftime("%Y-%m-%d %H:%M:%S", localtime())+" "+message)

def load(secure,hostname,url,schema,table,postdata,condition,verbose,rowcount):
  show("begin "+hostname+" "+url+" "+schema+" "+table+" "+(postdata or "")+" "+(condition or ""))
  if secure:
    address = "https://"+hostname+url
  else:
    address = "http://"+hostname+url
  show("load from "+address)

  reqheaders = {'Content-Type': 'application/json'}
  reqheaders['Caller-Id'] = '1.2.246.562.10.2013112012294919827487.vipunen'

  # api credentials from env vars
  if os.getenv("API_USERNAME"):
    show("using authentication")
    apiuser = os.getenv("API_USERNAME")
    apipass = os.getenv("API_PASSWORD")
    reqheaders['Authorization'] = 'Basic %s' % base64.b64encode(apiuser+":"+apipass)

  # automatic POST with (post)data
  print("value used for , -r, --rowcount=", rowcount)
  request = urllib.request.urlopen(address, data=postdata, headers=reqheaders)
  try:
    response = urllib.request.urlopen(request)
  except http.client.IncompleteRead as e:
    show('IncompleteRead exception.')
    show('Received: %d'%(e.partial))
    sys.exit(2)
  except urllib.error.HTTPError as e:
    show('The server couldn\'t fulfill the request.')
    show('Error code: %d'%(e.code))
    sys.exit(2)
  except urllib.error.URLError as e:
    show('We failed to reach a server.')
    show('Reason: %s'%(e.reason))
    sys.exit(2)
  else:
    # everything is fine
    show("api call OK")

  # remove data conditionally, otherwise empty
  # merge operation could be considered here...
  if condition:
    show("remove from %s.%s with condition '%s'"%(schema,table,condition))
    dboperator.execute("DELETE FROM %s.%s WHERE %s"%(schema,table,condition))
  else:
    show("empty %s.%s"%(schema,table))
    dboperator.empty(schema,table)

  show("insert data")
  cnt=0
  manycount = 0
  rows = []

  for row in ijson.items(response,'item'):
        cnt+=1
        manycount+=1
        # show some sign of being alive
        if cnt%100 == 0:
          sys.stdout.write('.')
          sys.stdout.flush()
        if cnt%1000 == 0:
          show("-- %d" % (cnt))
        if verbose: show("%d -- %s"%(cnt,row))

        # find out which columns to use on insert
        dboperator.resetcolumns(row)

        # flatten arrays/lists
        for col in row:
            if type(row[col]) is list:
                row[col] = ''.join(map(str,json.dumps(row[col])))
        rows.append(row)
        if cnt == 1:
            dboperator.insert(address,schema,table,row)
            manycount = 0
            rows = []
        if cnt > 1:
            if manycount == rowcount:
                insert(address,schema,table,rows)
                manycount = 0
                rows = []
  if len(rows) <= manycount and len(rows) > 0:
      insert(address,schema,table,rows)
      rows = []
      manycount = 0

  show("wrote %d"%(cnt))
  show("ready")

def insert(address,schema,table,rows):
    dboperator.insertMany(address,schema,table,rows)

def usage():
  print("""
usage: load.py [-s|--secure] -H|--hostname <hostname> -u|--url <url> -e|--schema <schema> -t|--table <table> [-p|--postdata] [-c|--condition <condition>] [-v|--verbose [-r]--rowcount]
""")

def main(argv):
  # muuttujat jotka kerrotaan argumentein
  secure=False
  hostname,url,schema,table="","","",""
  postdata=None
  condition=None
  verbose=False
  rowcount = 500

  try:
    opts,args=getopt.getopt(argv,"sH:u:e:t:p:c:v:r",["secure","hostname=","url=","schema=","table=","postdata=","condition=","verbose","rowcount="])
  except getopt.GetoptError as err:
    print(err)
    usage()
    sys.exit(2)
  for opt,arg in opts:
    if opt in ("-s", "--secure"): secure=True
    elif opt in ("-H", "--hostname"): hostname=arg
    elif opt in ("-u", "--url"): url=arg
    elif opt in ("-e", "--schema"): schema=arg
    elif opt in ("-t", "--table"): table=arg
    elif opt in ("-p", "--postdata"): postdata=arg
    elif opt in ("-c", "--condition"): condition=arg
    elif opt in ("-v", "--verbose"): verbose=True
    elif opt in ("-r", "--rowcount"): rowcount=arg
  if not hostname or not url or not schema or not table:
    usage()
    sys.exit(2)

  load(secure,hostname,url,schema,table,postdata,condition,verbose, rowcount)

  dboperator.close()

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>


<commit_before>import pandas as pd
import re
import yaml

class UnknownActionError(Exception):
    pass

class ActionList:
    def __init__(self, list_of_actions):
        self.actions = []
        for step in list_of_actions:
            try:
                action_name = list(step.keys())[0]
                instruction = step[action_name]
            except AttributeError:
                action_name = step
                instruction = []
            action = Action.factory(action_name, instruction)
            self.actions.append(action)

    def perform_instructions(self, input_data):
        output_data = input_data
        for action in self.actions:
            output_data = action.perform_instructions(output_data)
        return output_data

    def from_yaml(cls, filepath_or_buffer):
        if isinstance(filepath_or_buffer, str):
            with open(filepath_or_buffer) as f:
                actions = yaml.safe_load(f)
        else:
            actions = yaml.safe_load(filepath_or_buffer)
        return cls(actions)

# abstract, never used.
class Action:
    def __init__(self, instructions):
        self.instructions = instructions

    @staticmethod
    def factory(action, instruction):
        # just calls different constructors based on passed action
        action_classes = {
            'change_date_or_time_format': ChangeDateFormat,
            'copy_column': CopyAction,
            'rename_column': RenameAction,
            'extract_query_string': ExtractQueryStringAction,
            'extract_text': ExtractTextAction,
            'only_keep_these_columns': FilterColumnAction,
            'only_keep_rows_where': FilterRowAction,
            'only_edit_rows_where': EditSpecificRowsAction,
            'run_these_formula': FormulaAction,
            'remove_columns': RemoveColumnAction,
            'remove_duplicates': RemoveDuplicatesAction,
            'replace_text': ReplaceTextAction,
            'add_text_at_end': AppendTextAction,
            'add_text_at_start': PrependTextAction,
            'sum_up_by': GroupBySumAction,
            'make_column_names_lowercase': LowerCaseColumnNamesAction,
            'make_column_names_alphanumeric': AlphaNumColumnNamesAction,
            'change_format_of_column': ChangeColumnFormatAction,
        }
        try:
            action_class = action_classes[action]
            return action_class(instruction)
        except KeyError:
            msg = 'action {} unknown'.format(action)
            raise UnknownActionError(msg)

class GroupBySumAction(Action):
    """
    self.instructions: list of columns to group by.
    """
    def perform_instructions(self, input_data):
        columns_to_group_by = self.instructions
        grouped = input_data.groupby(columns_to_group_by, as_index=False)
        output_data = grouped.sum()
        return output_data

class ChangeDateFormat(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            date_format
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            result_col = instruction['result_column']
            date_format = instruction['date_format']
            date_format_strings = (
                # TODO: hour/minute/second support.
                ('YYYY', '%Y'),
                ('YY', '%y'),
                ('MM', '%m'),
                ('DD', '%d'),
            )
            for date_str, strformat in date_format_strings:
                date_format = date_format.replace(date_str, strformat)
                date_format = date_format.replace(date_str.lower(), strformat)
            format_date = lambda x: x.strftime(date_format)
            dates = input_data[instruction['target_column']].dropna()
            input_data[result_col] = dates.map(format_date)
        return input_data

class AlphaNumColumnNamesAction(Action):
    """
    """
    def perform_instructions(self, input_data):
        spaces_to_underscore = lambda x: x.replace(' ', '_')
        output_data = input_data.rename(columns=spaces_to_underscore)
        strip_punctuation = lambda x: re.sub('[^\w_]', '', x)
        output_data = output_data.rename(columns=strip_punctuation)
        return output_data

class LowerCaseColumnNamesAction(Action):
    """
    """
    def perform_instructions(self, input_data):
        output_data = input_data.rename(columns=str.lower)
        return output_data

class ChangeColumnFormatAction(Action):
    """
    self.instructions: list of dicts
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            column_format = list(instruction.values())[0]
            column = list(instruction.keys())[0]
            if column_format == 'date':
                output_data[column] = pd.to_datetime(output_data[column])
        return output_data

class AppendTextAction(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            text
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            col = instruction['target_column']
            text = instruction['text']
            result_col = instruction['result_column']
            # TODO: factor out commonality between Prepend/Append
            output_data[result_col] = output_data[col] + text
        return output_data

class PrependTextAction(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            text
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            col = instruction['target_column']
            text = instruction['text']
            result_col = instruction['result_column']
            output_data[result_col] = text + output_data[col]
        return output_data

class ReplaceTextAction(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            text_to_find
            replacement_text
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            col = instruction['target_column']
            replacement = instruction['replacement_text']
            text_to_find = instruction['text_to_find']
            if text_to_find == '^':
                result = replacement + output_data[col]
            elif text_to_find == '$':
                result = output_data[col] + replacement
            else:
                result = output_data[col].str.replace(text_to_find,
                                                      replacement)
            result_col = instruction['result_column']
            output_data[result_col] = result
        return output_data

class ExtractTextAction(Action):
    """
    self.instructions: dict
    keys:
        target_column
        result_column
        regex
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            text_col = instruction['target_column']
            result_col = instruction['result_column']
            regex = instruction['regex']
            text = input_data[text_col]
            input_data[result_col] = text.str.extract(regex, re.VERBOSE)
        return input_data

class ExtractQueryStringAction(Action):
    """
    self.instructions: list of dicts
    keys:
        result_column
        query_string
        target_column
    """
    def perform_instructions(self, input_data):
        regex_instructions = []
        for instruction in self.instructions:
            result_col = instruction['result_column']
            regex = """
                .*          # anything at the start
                [?&]        # query string prececed by & or ?
                {} =        # extract x query string.
                ([^&#]*)    # capture up until & or #
            """.format(instruction['query_string'])
            regex_instruction = {
                'target_column': instruction['target_column'],
                'result_column': result_col,
                'regex': regex,
            }
            regex_instructions.append(regex_instruction)
        a = ExtractTextAction(regex_instructions)
        return a.perform_instructions(input_data)

class FilterRowAction(Action):
    """
    self.instructions: list of strings with query
    e.g.
    'a < 2'
    '10 < temp < 20'
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            input_data = input_data.query(instruction)
        return input_data

class EditSpecificRowsAction(Action):
    """
    self.instructions: list of dict
        keys:
            filter_row
            run_these_formula
    e.g.
        - rows_match: 1 < b < 3
          run_these_formula: a = 666
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            filter_instruction = instruction['rows_match']
            filter_row_action = FilterRowAction([filter_instruction])
            filtered_data = filter_row_action.perform_instructions(output_data)
            actions = ActionList(instruction['list_of_actions'])
            transformed = actions.perform_instructions(filtered_data)
            output_data.loc[transformed.index] = transformed
        return input_data

class FilterColumnAction(Action):
    """
    self.instructions: list of columns to keep
    """
    def perform_instructions(self, input_data):
        output_data = input_data.loc[:, self.instructions]
        return output_data

class RemoveColumnAction(Action):
    """
    self.instructions: list of columns to drop
    """
    def perform_instructions(self, input_data):
        output_data = input_data.drop(self.instructions, axis='columns')
        return output_data

class RemoveDuplicatesAction(Action):
    """
    self.instructions: list of columns to drop duplicate values
    """
    def perform_instructions(self, input_data):
        output_data = input_data.drop_duplicates(self.instructions)
        return output_data

class RenameAction(Action):
    """
    self.instructions: list of strings of form:
    'new_column_name = old_column_name'
    """
    def perform_instructions(self, input_data):
        renames = {}
        for instruction in self.instructions:
            new_col, old_col = instruction.split('=')
            new_col, old_col = new_col.strip(), old_col.strip()
            renames[old_col] = new_col
        output_data = input_data.rename(columns=renames)
        return output_data

class CopyAction(Action):
    """
    self.instructions: list of strings of form:
    'new_column_name = old_column_name'
    """
    def perform_instructions(self, input_data):
        # TODO: refactor out 2 common lines? worth it?
        for instruction in self.instructions:
            new_col, old_col = instruction.split('=')
            new_col, old_col = new_col.strip(), old_col.strip()
            input_data[new_col] = input_data[old_col]
        return input_data

class FormulaAction(Action):
    """
    self.instructions: list of strings representing computation
    e.g.
    'fahrenheit = celsius * 9 / 5 + 32'
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            try:
                input_data.eval(instruction)
            except (ValueError, SyntaxError):
                # These hacks exists as .eval only works with numbers, not text
                result_col, value = instruction.split(' = ', maxsplit=1)
                if value[0] in ['"', "'"]:
                    # set to a string
                    input_data[result_col] = value[1:-1]
                else:
                    try:
                        input_data[result_col] = input_data[value]
                    except KeyError:
                        input_data[result_col] = value
        return input_data
<commit_msg>make ActionList.from_yaml classmethod<commit_after>import pandas as pd
import re
import yaml

class UnknownActionError(Exception):
    pass

class ActionList:
    def __init__(self, list_of_actions):
        self.actions = []
        for step in list_of_actions:
            try:
                action_name = list(step.keys())[0]
                instruction = step[action_name]
            except AttributeError:
                action_name = step
                instruction = []
            action = Action.factory(action_name, instruction)
            self.actions.append(action)

    def perform_instructions(self, input_data):
        output_data = input_data
        for action in self.actions:
            output_data = action.perform_instructions(output_data)
        return output_data

    @classmethod
    def from_yaml(cls, filepath_or_buffer):
        if isinstance(filepath_or_buffer, str):
            with open(filepath_or_buffer) as f:
                actions = yaml.safe_load(f)
        else:
            actions = yaml.safe_load(filepath_or_buffer)
        return cls(actions)

# abstract, never used.
class Action:
    def __init__(self, instructions):
        self.instructions = instructions

    @staticmethod
    def factory(action, instruction):
        # just calls different constructors based on passed action
        action_classes = {
            'change_date_or_time_format': ChangeDateFormat,
            'copy_column': CopyAction,
            'rename_column': RenameAction,
            'extract_query_string': ExtractQueryStringAction,
            'extract_text': ExtractTextAction,
            'only_keep_these_columns': FilterColumnAction,
            'only_keep_rows_where': FilterRowAction,
            'only_edit_rows_where': EditSpecificRowsAction,
            'run_these_formula': FormulaAction,
            'remove_columns': RemoveColumnAction,
            'remove_duplicates': RemoveDuplicatesAction,
            'replace_text': ReplaceTextAction,
            'add_text_at_end': AppendTextAction,
            'add_text_at_start': PrependTextAction,
            'sum_up_by': GroupBySumAction,
            'make_column_names_lowercase': LowerCaseColumnNamesAction,
            'make_column_names_alphanumeric': AlphaNumColumnNamesAction,
            'change_format_of_column': ChangeColumnFormatAction,
        }
        try:
            action_class = action_classes[action]
            return action_class(instruction)
        except KeyError:
            msg = 'action {} unknown'.format(action)
            raise UnknownActionError(msg)

class GroupBySumAction(Action):
    """
    self.instructions: list of columns to group by.
    """
    def perform_instructions(self, input_data):
        columns_to_group_by = self.instructions
        grouped = input_data.groupby(columns_to_group_by, as_index=False)
        output_data = grouped.sum()
        return output_data

class ChangeDateFormat(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            date_format
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            result_col = instruction['result_column']
            date_format = instruction['date_format']
            date_format_strings = (
                # TODO: hour/minute/second support.
                ('YYYY', '%Y'),
                ('YY', '%y'),
                ('MM', '%m'),
                ('DD', '%d'),
            )
            for date_str, strformat in date_format_strings:
                date_format = date_format.replace(date_str, strformat)
                date_format = date_format.replace(date_str.lower(), strformat)
            format_date = lambda x: x.strftime(date_format)
            dates = input_data[instruction['target_column']].dropna()
            input_data[result_col] = dates.map(format_date)
        return input_data

class AlphaNumColumnNamesAction(Action):
    """
    """
    def perform_instructions(self, input_data):
        spaces_to_underscore = lambda x: x.replace(' ', '_')
        output_data = input_data.rename(columns=spaces_to_underscore)
        strip_punctuation = lambda x: re.sub('[^\w_]', '', x)
        output_data = output_data.rename(columns=strip_punctuation)
        return output_data

class LowerCaseColumnNamesAction(Action):
    """
    """
    def perform_instructions(self, input_data):
        output_data = input_data.rename(columns=str.lower)
        return output_data

class ChangeColumnFormatAction(Action):
    """
    self.instructions: list of dicts
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            column_format = list(instruction.values())[0]
            column = list(instruction.keys())[0]
            if column_format == 'date':
                output_data[column] = pd.to_datetime(output_data[column])
        return output_data

class AppendTextAction(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            text
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            col = instruction['target_column']
            text = instruction['text']
            result_col = instruction['result_column']
            # TODO: factor out commonality between Prepend/Append
            output_data[result_col] = output_data[col] + text
        return output_data

class PrependTextAction(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            text
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            col = instruction['target_column']
            text = instruction['text']
            result_col = instruction['result_column']
            output_data[result_col] = text + output_data[col]
        return output_data

class ReplaceTextAction(Action):
    """
    self.instructions: list of dicts
        keys:
            target_column
            result_column
            text_to_find
            replacement_text
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            col = instruction['target_column']
            replacement = instruction['replacement_text']
            text_to_find = instruction['text_to_find']
            if text_to_find == '^':
                result = replacement + output_data[col]
            elif text_to_find == '$':
                result = output_data[col] + replacement
            else:
                result = output_data[col].str.replace(text_to_find,
                                                      replacement)
            result_col = instruction['result_column']
            output_data[result_col] = result
        return output_data

class ExtractTextAction(Action):
    """
    self.instructions: dict
    keys:
        target_column
        result_column
        regex
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            text_col = instruction['target_column']
            result_col = instruction['result_column']
            regex = instruction['regex']
            text = input_data[text_col]
            input_data[result_col] = text.str.extract(regex, re.VERBOSE)
        return input_data

class ExtractQueryStringAction(Action):
    """
    self.instructions: list of dicts
    keys:
        result_column
        query_string
        target_column
    """
    def perform_instructions(self, input_data):
        regex_instructions = []
        for instruction in self.instructions:
            result_col = instruction['result_column']
            regex = """
                .*          # anything at the start
                [?&]        # query string prececed by & or ?
                {} =        # extract x query string.
                ([^&#]*)    # capture up until & or #
            """.format(instruction['query_string'])
            regex_instruction = {
                'target_column': instruction['target_column'],
                'result_column': result_col,
                'regex': regex,
            }
            regex_instructions.append(regex_instruction)
        a = ExtractTextAction(regex_instructions)
        return a.perform_instructions(input_data)

class FilterRowAction(Action):
    """
    self.instructions: list of strings with query
    e.g.
    'a < 2'
    '10 < temp < 20'
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            input_data = input_data.query(instruction)
        return input_data

class EditSpecificRowsAction(Action):
    """
    self.instructions: list of dict
        keys:
            filter_row
            run_these_formula
    e.g.
        - rows_match: 1 < b < 3
          run_these_formula: a = 666
    """
    def perform_instructions(self, input_data):
        output_data = input_data
        for instruction in self.instructions:
            filter_instruction = instruction['rows_match']
            filter_row_action = FilterRowAction([filter_instruction])
            filtered_data = filter_row_action.perform_instructions(output_data)
            actions = ActionList(instruction['list_of_actions'])
            transformed = actions.perform_instructions(filtered_data)
            output_data.loc[transformed.index] = transformed
        return input_data

class FilterColumnAction(Action):
    """
    self.instructions: list of columns to keep
    """
    def perform_instructions(self, input_data):
        output_data = input_data.loc[:, self.instructions]
        return output_data

class RemoveColumnAction(Action):
    """
    self.instructions: list of columns to drop
    """
    def perform_instructions(self, input_data):
        output_data = input_data.drop(self.instructions, axis='columns')
        return output_data

class RemoveDuplicatesAction(Action):
    """
    self.instructions: list of columns to drop duplicate values
    """
    def perform_instructions(self, input_data):
        output_data = input_data.drop_duplicates(self.instructions)
        return output_data

class RenameAction(Action):
    """
    self.instructions: list of strings of form:
    'new_column_name = old_column_name'
    """
    def perform_instructions(self, input_data):
        renames = {}
        for instruction in self.instructions:
            new_col, old_col = instruction.split('=')
            new_col, old_col = new_col.strip(), old_col.strip()
            renames[old_col] = new_col
        output_data = input_data.rename(columns=renames)
        return output_data

class CopyAction(Action):
    """
    self.instructions: list of strings of form:
    'new_column_name = old_column_name'
    """
    def perform_instructions(self, input_data):
        # TODO: refactor out 2 common lines? worth it?
        for instruction in self.instructions:
            new_col, old_col = instruction.split('=')
            new_col, old_col = new_col.strip(), old_col.strip()
            input_data[new_col] = input_data[old_col]
        return input_data

class FormulaAction(Action):
    """
    self.instructions: list of strings representing computation
    e.g.
    'fahrenheit = celsius * 9 / 5 + 32'
    """
    def perform_instructions(self, input_data):
        for instruction in self.instructions:
            try:
                input_data.eval(instruction)
            except (ValueError, SyntaxError):
                # These hacks exists as .eval only works with numbers, not text
                result_col, value = instruction.split(' = ', maxsplit=1)
                if value[0] in ['"', "'"]:
                    # set to a string
                    input_data[result_col] = value[1:-1]
                else:
                    try:
                        input_data[result_col] = input_data[value]
                    except KeyError:
                        input_data[result_col] = value
        return input_data
<|endoftext|>