<commit_before># -*- coding: utf-8 -*-

"""
To run, type :code:`python3 -m pybel_tools.analysis.neurommsig.export` in the command line
"""

import itertools as itt
import logging

import os
import pandas as pd
import re
from functools import partial

from pybel.utils import ensure_quotes, get_bel_resource
from ...document_utils import write_boilerplate
from ...resources import DBSNP_PATTERN, NIFT, HGNC_HUMAN_GENES, NEUROMMSIG, MESHD

log = logging.getLogger(__name__)

HGNCsymbolpattern = re.compile("^[A-Z0-9-]+$|^C[0-9XY]+orf[0-9]+$")

SNPpattern = re.compile("^rs[0-9]+$")
SNPspatternSpace = re.compile("^(rs[0-9]+)\s((rs[0-9]+)\s)*(rs[0-9]+)$")
SNPspatternComma = re.compile("^(rs[0-9]+),((rs[0-9]+),)*(rs[0-9]+)$")
SNPspatternSpaceComma = re.compile("^(rs[0-9]+), ((rs[0-9]+), )*(rs[0-9]+)$")
Checked_by_Anandhi = re.compile("No")

miRNApattern = re.compile("^MIR.*$")
miRNAspattern = re.compile("^(MIR.*),((MIR.*$),)*(MIR.*$)$")

def preprocessing_excel(filepath):
    """Preprocessing of the excel file.

    Parameters
    ----------
    filepath : Filepath of the excel sheet

    Returns
    -------
    dataframe : Preprocessed pandas dataframe

    """

    if not os.path.exists(filepath):
        raise ValueError("Error: %s file not found" % filepath)

    # Import Models from Excel sheet, independent for AD and PD
    df = pd.read_excel(filepath, sheetname=0, header=0)

    # Indexes and column name
    # [log.info(str(x)+': '+str((df.columns.values[x]))) for x in range (0,len(df.columns.values))]

    # Starting from 4: Pathway Name

    # Fill Pathway cells that are merged and are 'NaN' after deleting rows where there is no genes
    df.iloc[:, 4] = pd.Series(df.iloc[:, 4]).fillna(method='ffill')

    # Number of gaps
    # log.info(df.ix[:,6].isnull().sum())

    df = df[df.ix[:, 6].notnull()]
    df = df.reset_index(drop=True)

    # Fill NaN to ceros in PubmedID column
    df.ix[:, 7].fillna(0, inplace=True)

    # Number of gaps in the gene column should be already zero
    if (df.ix[:, 6].isnull().sum()) != 0:
        raise ValueError("Error: Empty cells in the gene column")

    # Check current state
    # df.to_csv('out.csv')

    return df

def munge_cell(cell, line=None, validators=None):
    """

    :param cell:
    :param line:
    :param validators:
    :return:
    """
    if pd.isnull(cell):
        return None

    c = ' '.join(cell.split())

    if validators is not None and all(re.match(validator, c) is None for validator in validators):
        if line:
            log.info("Munge cell error: aprox in line: %s: %s", line, c)
        return None

    return [x.strip() for x in str(c).strip().split(',')]

def preprocessing_br_projection_excel(filepath):
    """Preprocessing of the excel file.

    Parameters
    ----------
    filepath : Filepath of the excel sheet

    Returns
    -------
    dataframe : Preprocessed pandas dataframe

    """

    if not os.path.exists(filepath):
        raise ValueError("Error: %s file not found" % filepath)

    df = pd.read_excel(filepath, sheetname=0, header=0)

    return df

munge_snp = partial(munge_cell, validators=[SNPpattern, SNPspatternSpaceComma])

mesh_alzheimer = "Alzheimer Disease"  # Death to the eponym!
mesh_parkinson = "Parkinson Disease"

pathway_column = 'Subgraph Name'
columns = [
    'Genes',
    'PMIDs',
    'SNPs from Literature (Aybuge)',
    'Genome wide associated SNPs (Mufassra)',
    'LD block analysis (Mufassra)',
    'Imaging Features (Anandhi)',
    'SNP_Image Feature (Mufassra & Anandhi)',
]

def preprocess(path):
    """

    :param str path:
    :return:
    :rtype: pandas.DataFrame
    """
    df = preprocessing_excel(path)
    df['SNPs from Literature (Aybuge)'] = df['SNPs from Literature (Aybuge)'].map(munge_snp)
    df['Genome wide associated SNPs (Mufassra)'] = df['Genome wide associated SNPs (Mufassra)'].map(munge_snp)
    df['LD block analysis (Mufassra)'] = df['LD block analysis (Mufassra)'].map(munge_snp)
    df['Imaging Features (Anandhi)'] = df['Imaging Features (Anandhi)'].map(munge_cell)
    df['Imaging Features (Anandhi)'] = df['Imaging Features (Anandhi)'].map(
        lambda c: None if c is not None and c[0] == 'No' else c)
    df['SNP_Image Feature (Mufassra & Anandhi)'] = df['SNP_Image Feature (Mufassra & Anandhi)'].map(munge_snp)
    return df

def get_nift_values():
    """

    :return:
    :rtype: dict[str,str]
    """
    r = get_bel_resource(NIFT)
    return {v.lower(): v for v in r['Values']}

def write_neurommsig_bel(file, df, disease, nift_values):
    """Writes the NeuroMMSigDB excel sheet to BEL

    :param file: a file or file-like that can be writen to
    :param pandas.DataFrame df:
    :param str disease:
    :param dict nift_values: a dictionary of lowercased to normal names in NIFT
    """
    write_boilerplate(
        document_name='NeuroMMSigDB for {}'.format(disease),
        description='SNP and Clinical Features for Subgraphs in {}'.format(disease),
        authors='Daniel Domingo, Charles Tapley Hoyt, Mufassra Naz, Aybuge Altay, Anandhi Iyappan',
        contact='charles.hoyt@scai.fraunhofer.de',
        namespace_dict={
            'NIFT': NIFT,
            'HGNC': HGNC_HUMAN_GENES,
        },
        namespace_patterns={
            'dbSNP': DBSNP_PATTERN
        },
        annotations_dict={
            'Subgraph': NEUROMMSIG,
            'MeSHDisease': MESHD
        },
        file=file
    )

    print('SET Citation = {"URL", "NeuroMMSigDB", "http://neurommsig.scai.fraunhofer.de/"}', file=file)
    print('SET Evidence = "Serialized from NeuroMMSigDB"', file=file)
    print('SET MeSHDisease = "{}"\n'.format(disease), file=file)

    missing_features = set()
    fixed_caps = set()
    nift_value_originals = set(nift_values.values())

    for pathway, pathway_df in df.groupby(pathway_column):
        print('SET Subgraph = "{}"'.format(pathway), file=file)

        for _, gene, pubmeds, lit_snps, gwas_snps, ld_block_snps, clinical_features, clinical_snp in pathway_df[
            columns].itertuples():
            gene = ensure_quotes(gene)

            if lit_snps is None:
                lit_snps = []

            if gwas_snps is None:
                gwas_snps = []

            if clinical_snp is None:
                clinical_snp = []

            for snp in itt.chain(lit_snps, gwas_snps, clinical_snp):
                if not snp.strip():
                    continue
                print('g(HGNC:{}) -- g(dbSNP:{})'.format(gene, snp), file=file)

            for clinical_feature in clinical_features or []:
                if not clinical_feature.strip():
                    continue
                if clinical_feature.lower() not in nift_values:
                    missing_features.add(clinical_feature)
                    continue
                if clinical_feature not in nift_value_originals:
                    fixed_caps.add((clinical_feature, nift_values[clinical_feature.lower()]))
                    clinical_feature = nift_values[clinical_feature.lower()]  # fix capitalization
                print('g(HGNC:{}) -- a(NIFT:{})'.format(gene, ensure_quotes(clinical_feature)), file=file)

        print('UNSET Subgraph\n', file=file)

    print('UNSET MeSHDisease', file=file)
    print('UNSET Evidence', file=file)
    print('UNSET Citation', file=file)

    log.warning('Missing Features in %s', disease)
    for feature in missing_features:
        log.warning(feature)

    log.warning('Fixed capitalization')
    for broken, fixed in fixed_caps:
        log.warning('%s -> %s', broken, fixed)

if __name__ == '__main__':
    bms_base = os.environ['BMS_BASE']
    neurommsig_base = os.environ['NEUROMMSIG_BASE']
    neurommsig_excel_dir = os.path.join(neurommsig_base, 'resources', 'excels')

    nift_values = get_nift_values()

    ad_path = os.path.join(neurommsig_excel_dir, 'AD.xlsx')
    ad_df = preprocess(ad_path)
    with open(os.path.join(bms_base, 'aetionomy', 'alzheimers', 'neurommsigdb_ad.bel'), 'w') as ad_file:
        write_neurommsig_bel(ad_file, ad_df, mesh_alzheimer, nift_values)

    pd_path = os.path.join(neurommsig_excel_dir, 'PD.xlsx')
    pd_df = preprocess(pd_path)
    with open(os.path.join(bms_base, 'aetionomy', 'parkinsons', 'neurommsigdb_pd.bel'), 'w') as pd_file:
        write_neurommsig_bel(pd_file, pd_df, mesh_parkinson, nift_values)
<commit_msg>update neurommsig bel exporter<commit_after># -*- coding: utf-8 -*-

"""
To run, type :code:`python3 -m pybel_tools.analysis.neurommsig.export` in the command line
"""

import itertools as itt
import logging

import os
import pandas as pd
import re
from functools import partial

from pybel.utils import ensure_quotes, get_bel_resource
from ...document_utils import write_boilerplate
from ...resources import DBSNP_PATTERN, NIFT, HGNC_HUMAN_GENES, NEUROMMSIG, MESHD

log = logging.getLogger(__name__)

HGNCsymbolpattern = re.compile("^[A-Z0-9-]+$|^C[0-9XY]+orf[0-9]+$")

SNPpattern = re.compile("^rs[0-9]+$")
SNPspatternSpace = re.compile("^(rs[0-9]+)\s((rs[0-9]+)\s)*(rs[0-9]+)$")
SNPspatternComma = re.compile("^(rs[0-9]+),((rs[0-9]+),)*(rs[0-9]+)$")
SNPspatternSpaceComma = re.compile("^(rs[0-9]+), ((rs[0-9]+), )*(rs[0-9]+)$")
Checked_by_Anandhi = re.compile("No")

miRNApattern = re.compile("^MIR.*$")
miRNAspattern = re.compile("^(MIR.*),((MIR.*$),)*(MIR.*$)$")

def preprocessing_excel(filepath):
    """Preprocessing of the excel file.

    Parameters
    ----------
    filepath : Filepath of the excel sheet

    Returns
    -------
    dataframe : Preprocessed pandas dataframe

    """

    if not os.path.exists(filepath):
        raise ValueError("Error: %s file not found" % filepath)

    # Import Models from Excel sheet, independent for AD and PD
    df = pd.read_excel(filepath, sheetname=0, header=0)

    # Indexes and column name
    # [log.info(str(x)+': '+str((df.columns.values[x]))) for x in range (0,len(df.columns.values))]

    # Starting from 4: Pathway Name

    # Fill Pathway cells that are merged and are 'NaN' after deleting rows where there is no genes
    df.iloc[:, 4] = pd.Series(df.iloc[:, 4]).fillna(method='ffill')

    # Number of gaps
    # log.info(df.ix[:,6].isnull().sum())

    df = df[df.ix[:, 6].notnull()]
    df = df.reset_index(drop=True)

    # Fill NaN to ceros in PubmedID column
    df.ix[:, 7].fillna(0, inplace=True)

    # Number of gaps in the gene column should be already zero
    if (df.ix[:, 6].isnull().sum()) != 0:
        raise ValueError("Error: Empty cells in the gene column")

    # Check current state
    # df.to_csv('out.csv')

    return df

def munge_cell(cell, line=None, validators=None):
    """

    :param cell:
    :param line:
    :param validators:
    :return:
    """
    if pd.isnull(cell):
        return None

    c = ' '.join(cell.split())

    if validators is not None and all(re.match(validator, c) is None for validator in validators):
        if line:
            log.info("Munge cell error: aprox in line: %s: %s", line, c)
        return None

    return [x.strip() for x in str(c).strip().split(',')]

def preprocessing_br_projection_excel(filepath):
    """Preprocessing of the excel file.

    Parameters
    ----------
    filepath : Filepath of the excel sheet

    Returns
    -------
    dataframe : Preprocessed pandas dataframe

    """

    if not os.path.exists(filepath):
        raise ValueError("Error: %s file not found" % filepath)

    df = pd.read_excel(filepath, sheetname=0, header=0)

    return df

munge_snp = partial(munge_cell, validators=[SNPpattern, SNPspatternSpaceComma])

mesh_alzheimer = "Alzheimer Disease"  # Death to the eponym!
mesh_parkinson = "Parkinson Disease"

pathway_column = 'Subgraph Name'
columns = [
    'Genes',
    'PMIDs',
    'SNPs from Literature (Aybuge)',
    'Genome wide associated SNPs (Mufassra)',
    'LD block analysis (Mufassra)',
    'Imaging Features (Anandhi)',
    'SNP_Image Feature (Mufassra & Anandhi)',
]

def preprocess(path):
    """

    :param str path:
    :return:
    :rtype: pandas.DataFrame
    """
    df = preprocessing_excel(path)
    df['SNPs from Literature (Aybuge)'] = df['SNPs from Literature (Aybuge)'].map(munge_snp)
    df['Genome wide associated SNPs (Mufassra)'] = df['Genome wide associated SNPs (Mufassra)'].map(munge_snp)
    df['LD block analysis (Mufassra)'] = df['LD block analysis (Mufassra)'].map(munge_snp)
    df['Imaging Features (Anandhi)'] = df['Imaging Features (Anandhi)'].map(munge_cell)
    df['Imaging Features (Anandhi)'] = df['Imaging Features (Anandhi)'].map(
        lambda c: None if c is not None and c[0] == 'No' else c)
    df['SNP_Image Feature (Mufassra & Anandhi)'] = df['SNP_Image Feature (Mufassra & Anandhi)'].map(munge_snp)
    return df

def get_nift_values():
    """

    :return:
    :rtype: dict[str,str]
    """
    r = get_bel_resource(NIFT)
    return {v.lower(): v for v in r['Values']}

def write_neurommsig_bel(file, df, disease, nift_values):
    """Writes the NeuroMMSigDB excel sheet to BEL

    :param file: a file or file-like that can be writen to
    :param pandas.DataFrame df:
    :param str disease:
    :param dict nift_values: a dictionary of lowercased to normal names in NIFT
    """
    write_boilerplate(
        document_name='NeuroMMSigDB for {}'.format(disease),
        description='SNP and Clinical Features for Subgraphs in {}'.format(disease),
        authors='Daniel Domingo, Charles Tapley Hoyt, Mufassra Naz, Aybuge Altay, Anandhi Iyappan',
        contact='charles.hoyt@scai.fraunhofer.de',
        namespace_dict={
            'NIFT': NIFT,
            'HGNC': HGNC_HUMAN_GENES,
        },
        namespace_patterns={
            'dbSNP': DBSNP_PATTERN
        },
        annotations_dict={
            'Subgraph': NEUROMMSIG,
            'MeSHDisease': MESHD
        },
        file=file
    )

    print('SET Citation = {"URL", "NeuroMMSigDB", "http://neurommsig.scai.fraunhofer.de/"}', file=file)
    print('SET Evidence = "Serialized from NeuroMMSigDB"', file=file)
    print('SET MeSHDisease = "{}"\n'.format(disease), file=file)

    missing_features = set()
    fixed_caps = set()
    nift_value_originals = set(nift_values.values())

    for pathway, pathway_df in df.groupby(pathway_column):
        print('SET Subgraph = "{}"'.format(pathway), file=file)

        for _, gene, pubmeds, lit_snps, gwas_snps, ld_block_snps, clinical_features, clinical_snp in pathway_df[
            columns].itertuples():
            gene = ensure_quotes(gene)

            if lit_snps is None:
                lit_snps = []

            # TODO: Stick PubMeds into the evidence somehow
            if pubmeds is None:
                pubmeds = []

            if ld_block_snps is None:
                ld_block_snps = []

            if gwas_snps is None:
                gwas_snps = []

            if clinical_snp is None:
                clinical_snp = []

            for snp in itt.chain(lit_snps, gwas_snps,ld_block_snps, clinical_snp):
                if not snp.strip():
                    continue
                print('g(HGNC:{}) -- g(dbSNP:{})'.format(gene, snp), file=file)

            for clinical_feature in clinical_features or []:
                if not clinical_feature.strip():
                    continue
                if clinical_feature.lower() not in nift_values:
                    missing_features.add(clinical_feature)
                    continue
                if clinical_feature not in nift_value_originals:
                    fixed_caps.add((clinical_feature, nift_values[clinical_feature.lower()]))
                    clinical_feature = nift_values[clinical_feature.lower()]  # fix capitalization
                print('g(HGNC:{}) -- a(NIFT:{})'.format(gene, ensure_quotes(clinical_feature)), file=file)

        print('UNSET Subgraph\n', file=file)

    print('UNSET MeSHDisease', file=file)
    print('UNSET Evidence', file=file)
    print('UNSET Citation', file=file)

    log.warning('Missing Features in %s', disease)
    for feature in missing_features:
        log.warning(feature)

    log.warning('Fixed capitalization')
    for broken, fixed in fixed_caps:
        log.warning('%s -> %s', broken, fixed)

if __name__ == '__main__':
    bms_base = os.environ['BMS_BASE']
    neurommsig_base = os.environ['NEUROMMSIG_BASE']
    neurommsig_excel_dir = os.path.join(neurommsig_base, 'resources', 'excels')

    nift_values = get_nift_values()

    ad_path = os.path.join(neurommsig_excel_dir, 'AD.xlsx')
    ad_df = preprocess(ad_path)
    with open(os.path.join(bms_base, 'aetionomy', 'alzheimers', 'neurommsigdb_ad.bel'), 'w') as ad_file:
        write_neurommsig_bel(ad_file, ad_df, mesh_alzheimer, nift_values)

    pd_path = os.path.join(neurommsig_excel_dir, 'PD.xlsx')
    pd_df = preprocess(pd_path)
    with open(os.path.join(bms_base, 'aetionomy', 'parkinsons', 'neurommsigdb_pd.bel'), 'w') as pd_file:
        write_neurommsig_bel(pd_file, pd_df, mesh_parkinson, nift_values)
<|endoftext|>


<commit_before># Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import textwrap
from dataclasses import dataclass
from typing import Iterable, Mapping

from pants.backend.go.subsystems.golang import GolangSubsystem
from pants.backend.go.util_rules import goroot
from pants.backend.go.util_rules.goroot import GoRoot
from pants.core.util_rules.system_binaries import BashBinary
from pants.engine.environment import Environment, EnvironmentRequest
from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent, MergeDigests
from pants.engine.internals.selectors import Get, MultiGet
from pants.engine.platform import Platform
from pants.engine.process import Process, ProcessResult
from pants.engine.rules import collect_rules, rule
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.meta import frozen_after_init

@frozen_after_init
@dataclass(unsafe_hash=True)
class GoSdkProcess:
    command: tuple[str, ...]
    description: str
    env: FrozenDict[str, str]
    input_digest: Digest
    working_dir: str | None
    output_files: tuple[str, ...]
    output_directories: tuple[str, ...]
    platform: Platform | None

    def __init__(
        self,
        command: Iterable[str],
        *,
        description: str,
        env: Mapping[str, str] | None = None,
        input_digest: Digest = EMPTY_DIGEST,
        working_dir: str | None = None,
        output_files: Iterable[str] = (),
        output_directories: Iterable[str] = (),
        allow_downloads: bool = False,
        platform: Platform | None = None,
    ) -> None:
        self.command = tuple(command)
        self.description = description
        self.env = (
            FrozenDict(env or {})
            if allow_downloads
            else FrozenDict({**(env or {}), "GOPROXY": "off"})
        )
        self.input_digest = input_digest
        self.working_dir = working_dir
        self.output_files = tuple(output_files)
        self.output_directories = tuple(output_directories)
        self.platform = platform

@dataclass(frozen=True)
class GoSdkRunSetup:
    digest: Digest
    script: FileContent

    CHDIR_ENV = "__PANTS_CHDIR_TO"

@rule
async def go_sdk_invoke_setup(goroot: GoRoot) -> GoSdkRunSetup:
    # Note: The `go` tool requires GOPATH to be an absolute path which can only be resolved
    # from within the execution sandbox. Thus, this code uses a bash script to be able to resolve
    # absolute paths inside the sandbox.
    go_run_script = FileContent(
        "__run_go.sh",
        textwrap.dedent(
            f"""\
            export GOROOT={goroot.path}
            export GOPATH="$(/bin/pwd)/gopath"
            export GOCACHE="$(/bin/pwd)/cache"
            /bin/mkdir -p "$GOPATH" "$GOCACHE"
            if [ -n "${GoSdkRunSetup.CHDIR_ENV}" ]; then
              cd "${GoSdkRunSetup.CHDIR_ENV}"
            fi
            exec "{goroot.path}/bin/go" "$@"
            """
        ).encode("utf-8"),
    )

    digest = await Get(Digest, CreateDigest([go_run_script]))
    return GoSdkRunSetup(digest, go_run_script)

@rule
async def setup_go_sdk_process(
    request: GoSdkProcess,
    go_sdk_run: GoSdkRunSetup,
    bash: BashBinary,
    golang_subsystem: GolangSubsystem,
    goroot: GoRoot,
) -> Process:
    input_digest, env_vars = await MultiGet(
        Get(Digest, MergeDigests([go_sdk_run.digest, request.input_digest])),
        Get(Environment, EnvironmentRequest(golang_subsystem.env_vars_to_pass_to_subprocesses)),
    )
    return Process(
        argv=[bash.path, go_sdk_run.script.path, *request.command],
        env={
            **env_vars,
            **request.env,
            GoSdkRunSetup.CHDIR_ENV: request.working_dir or "",
            # TODO: Maybe could just use MAJOR.MINOR for version part here?
            "__PANTS_GO_SDK_CACHE_KEY": f"{goroot.version}/{goroot.goos}/{goroot.goarch}",
        },
        input_digest=input_digest,
        description=request.description,
        output_files=request.output_files,
        output_directories=request.output_directories,
        level=LogLevel.DEBUG,
        platform=request.platform,
    )

@dataclass(frozen=True)
class GoSdkToolIDRequest:
    tool_name: str

@dataclass(frozen=True)
class GoSdkToolIDResult:
    tool_name: str
    tool_id: str

@rule
async def compute_go_tool_id(request: GoSdkToolIDRequest) -> GoSdkToolIDResult:
    result = await Get(
        ProcessResult,
        GoSdkProcess(
            ["tool", request.tool_name, "-V=full"],
            description=f"Obtain tool ID for Go tool `{request.tool_name}`.",
        ),
    )
    return GoSdkToolIDResult(tool_name=request.tool_name, tool_id=result.stdout.decode().strip())

def rules():
    return (*collect_rules(), *goroot.rules())
<commit_msg>go: provide a way to inject sandbox root into args (#16722)<commit_after># Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import textwrap
from dataclasses import dataclass
from typing import Iterable, Mapping

from pants.backend.go.subsystems.golang import GolangSubsystem
from pants.backend.go.util_rules import goroot
from pants.backend.go.util_rules.goroot import GoRoot
from pants.core.util_rules.system_binaries import BashBinary
from pants.engine.environment import Environment, EnvironmentRequest
from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent, MergeDigests
from pants.engine.internals.selectors import Get, MultiGet
from pants.engine.platform import Platform
from pants.engine.process import Process, ProcessResult
from pants.engine.rules import collect_rules, rule
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.meta import frozen_after_init

@frozen_after_init
@dataclass(unsafe_hash=True)
class GoSdkProcess:
    command: tuple[str, ...]
    description: str
    env: FrozenDict[str, str]
    input_digest: Digest
    working_dir: str | None
    output_files: tuple[str, ...]
    output_directories: tuple[str, ...]
    platform: Platform | None
    replace_sandbox_root_in_args: bool

    def __init__(
        self,
        command: Iterable[str],
        *,
        description: str,
        env: Mapping[str, str] | None = None,
        input_digest: Digest = EMPTY_DIGEST,
        working_dir: str | None = None,
        output_files: Iterable[str] = (),
        output_directories: Iterable[str] = (),
        allow_downloads: bool = False,
        platform: Platform | None = None,
        replace_sandbox_root_in_args: bool = False,
    ) -> None:
        self.command = tuple(command)
        self.description = description
        self.env = (
            FrozenDict(env or {})
            if allow_downloads
            else FrozenDict({**(env or {}), "GOPROXY": "off"})
        )
        self.input_digest = input_digest
        self.working_dir = working_dir
        self.output_files = tuple(output_files)
        self.output_directories = tuple(output_directories)
        self.platform = platform
        self.replace_sandbox_root_in_args = replace_sandbox_root_in_args

@dataclass(frozen=True)
class GoSdkRunSetup:
    digest: Digest
    script: FileContent

    CHDIR_ENV = "__PANTS_CHDIR_TO"
    SANDBOX_ROOT_ENV = "__PANTS_REPLACE_SANDBOX_ROOT"

@rule
async def go_sdk_invoke_setup(goroot: GoRoot) -> GoSdkRunSetup:
    # Note: The `go` tool requires GOPATH to be an absolute path which can only be resolved
    # from within the execution sandbox. Thus, this code uses a bash script to be able to resolve
    # absolute paths inside the sandbox.
    go_run_script = FileContent(
        "__run_go.sh",
        textwrap.dedent(
            f"""\
            export GOROOT={goroot.path}
            sandbox_root="$(/bin/pwd)"
            export GOPATH="${{sandbox_root}}/gopath"
            export GOCACHE="${{sandbox_root}}/cache"
            /bin/mkdir -p "$GOPATH" "$GOCACHE"
            if [ -n "${GoSdkRunSetup.CHDIR_ENV}" ]; then
              cd "${GoSdkRunSetup.CHDIR_ENV}"
            fi
            if [ -n "${GoSdkRunSetup.SANDBOX_ROOT_ENV}" ]; then
              args=("${{@//__PANTS_SANDBOX_ROOT__/$sandbox_root}}")
              set -- "${{args[@]}}"
            fi
            exec "{goroot.path}/bin/go" "$@"
            """
        ).encode("utf-8"),
    )

    digest = await Get(Digest, CreateDigest([go_run_script]))
    return GoSdkRunSetup(digest, go_run_script)

@rule
async def setup_go_sdk_process(
    request: GoSdkProcess,
    go_sdk_run: GoSdkRunSetup,
    bash: BashBinary,
    golang_subsystem: GolangSubsystem,
    goroot: GoRoot,
) -> Process:
    input_digest, env_vars = await MultiGet(
        Get(Digest, MergeDigests([go_sdk_run.digest, request.input_digest])),
        Get(Environment, EnvironmentRequest(golang_subsystem.env_vars_to_pass_to_subprocesses)),
    )
    maybe_replace_sandbox_root_env = (
        {GoSdkRunSetup.SANDBOX_ROOT_ENV: "1"} if request.replace_sandbox_root_in_args else {}
    )
    return Process(
        argv=[bash.path, go_sdk_run.script.path, *request.command],
        env={
            **env_vars,
            **request.env,
            GoSdkRunSetup.CHDIR_ENV: request.working_dir or "",
            **maybe_replace_sandbox_root_env,
            # TODO: Maybe could just use MAJOR.MINOR for version part here?
            "__PANTS_GO_SDK_CACHE_KEY": f"{goroot.version}/{goroot.goos}/{goroot.goarch}",
        },
        input_digest=input_digest,
        description=request.description,
        output_files=request.output_files,
        output_directories=request.output_directories,
        level=LogLevel.DEBUG,
        platform=request.platform,
    )

@dataclass(frozen=True)
class GoSdkToolIDRequest:
    tool_name: str

@dataclass(frozen=True)
class GoSdkToolIDResult:
    tool_name: str
    tool_id: str

@rule
async def compute_go_tool_id(request: GoSdkToolIDRequest) -> GoSdkToolIDResult:
    result = await Get(
        ProcessResult,
        GoSdkProcess(
            ["tool", request.tool_name, "-V=full"],
            description=f"Obtain tool ID for Go tool `{request.tool_name}`.",
        ),
    )
    return GoSdkToolIDResult(tool_name=request.tool_name, tool_id=result.stdout.decode().strip())

def rules():
    return (*collect_rules(), *goroot.rules())
<|endoftext|>


<commit_before>'''
Created on Dec 4, 2012

Module to handle dynamic decoration of LCIO classes

@author: Christian Grefe
'''

from __future__ import absolute_import, unicode_literals
from ROOT import TVector3, TLorentzVector
from pyLCIO import EVENT, IMPL, IO, UTIL
from array import array

def addMethod( self, method, name=None ):
    ''' Convenience method to add a method to a class '''
    if not name:
        name = method.func_name
    setattr( self, name, method )

def getEndpointVec( self ):
    ''' Wrapper for the getEndpoint method to return a TVector3 '''
    v = self.getEndpoint()
    return TVector3( v[0], v[1], v[2] )

def getLorentzVec( self ):
    ''' Convenience method for classes that have momentum and energy '''
    energy = self.getEnergy()
    v = self.getMomentum()
    return TLorentzVector( v[0], v[1], v[2], energy )

def getMomentumVec( self ):
    ''' Wrapper for the getMomentum method to return a TVector3 '''
    v = self.getMomentum()
    return TVector3( v[0], v[1], v[2] )

def setMomentumVec( self, v ):
    ''' Wrapper for the setMomentum method passing a vector '''
    self.setMomentum( array( 'f', [v.x(), v.y(), v.z()] ) )

def getPositionVec( self ):
    ''' Wrapper for the getPosition method to return a TVector3 '''
    v = self.getPosition()
    return TVector3( v[0], v[1], v[2] )

def setPositionVec( self, v ):
    ''' Wrapper for the setPosition method passing a vector '''
    # tracker hits take pointer to doubles, calorimeter hits pointer to floats
    try:
        self.setPosition( array( 'd', [v.x(), v.y(), v.z()] ) )
    except TypeError:
        self.setPosition( array( 'f', [v.x(), v.y(), v.z()] ) )

def setRawDataVec( self, v ):
    ''' Wrapper for the setRawData method passing a vector '''
    self.setRawData( array( 'i', v ), len(v) )

def getReferencePointVec( self ):
    ''' Wrapper for the getReferencePoint method to return a TVector3 '''
    v = self.getReferencePoint()
    return TVector3( v[0], v[1], v[2] )

def setReferencePointVec( self, v ):
    ''' Wrapper for the setReferencePoint method passing a vector '''
    self.setReferencePoint( array( 'f', [v.x(), v.y(), v.z()] ) )

def getVertexVec( self ):
    ''' Wrapper for the getVertex method to return a TVector3 '''
    v = self.getVertex()
    return TVector3( v[0], v[1], v[2] )

def setVertexVec( self, v ):
    ''' Wrapper for the setVertex method passing a vector '''
    self.setVertex( array( 'd', [v.x(), v.y(), v.z()] ) )

def LCCollectionIterator( self ):
    ''' Helper method to make LCCollection iterable '''
    index = 0
    nElements = self.getNumberOfElements()
    while index < nElements:
        yield self.getElementAt( index )
        index += 1

def LCEventIterator( self ):
    ''' Helper method to make LCEvent iterable '''
    for collectionName in self.getCollectionNames():
        yield collectionName, self.getCollection( collectionName )

def LCReaderIterator( self ):
    ''' Helper method to make LCReader iterable '''
    event = self.readNextEvent()
    while event:
        yield event
        event = self.readNextEvent()

def LCStdHepRdrIterator( self ):
    ''' Helper method to make LCStdHepRdr iterable '''
    processedEvents = 0
    while True:
        event = IMPL.LCEventImpl()
        try:
            self.updateNextEvent( event )
            event.setEventNumber( processedEvents )
        except Exception:
            break
        yield event
        processedEvents += 1

def getMcParticles( self ):
    ''' Convenience method to get the default mc particle collection '''
    return self.getCollection( EVENT.LCIO.MCPARTICLE )

def getTracks( self ):
    ''' Convenience method to get the default track collection '''
    return self.getCollection( EVENT.LCIO.TRACK )

# List of LCIO classes to decorate
lcioClasses = [ EVENT.LCEvent,
                EVENT.CalorimeterHit,
                EVENT.Cluster,
                EVENT.RawCalorimeterHit,
                EVENT.ReconstructedParticle,
                EVENT.SimCalorimeterHit,
                EVENT.SimTrackerHit,
                EVENT.TPCHit,
                EVENT.Track,
                EVENT.TrackerData,
                EVENT.TrackerHit,
                EVENT.TrackState,
                EVENT.Vertex,
                EVENT.MCParticle,
                EVENT.LCCollection,
                IMPL.MCParticleImpl,
                IMPL.CalorimeterHitImpl,
                IMPL.ClusterImpl,
                IMPL.RawCalorimeterHitImpl,
                IMPL.ReconstructedParticleImpl,
                IMPL.SimCalorimeterHitImpl,
                IMPL.SimTrackerHitImpl,
                IMPL.TPCHitImpl,
                IMPL.TrackerDataImpl,
                IMPL.TrackerHitImpl,
                IMPL.TrackerHitPlaneImpl,
                IMPL.TrackerHitZCylinderImpl,
                IMPL.TrackerPulseImpl,
                IMPL.TrackerRawDataImpl,
                IMPL.TrackImpl,
                IMPL.TrackStateImpl,
                IMPL.VertexImpl,
                IO.LCReader,
                UTIL.LCStdHepRdr,
                ]

# Cache decorated classes
decoratedClasses = []

def decorateClass( o ):
    ''' Method to wrap the decoration logic '''
    if o not in decoratedClasses:
        d = dir( o )
        if 'getEndpoint' in d:
            addMethod( o, getEndpointVec )
        if 'getMomentum' in d:
            addMethod( o, getMomentumVec )
        if 'setMomentum' in d:
            addMethod( o, setMomentumVec )
        if 'getMomentum' and 'getEnergy' in d:
            addMethod( o, getLorentzVec )
        if 'getPosition' in d:
            addMethod( o, getPositionVec )
        if 'setPosition' in d:
            addMethod( o, setPositionVec )
        if 'getReferencePoint' in d:
            addMethod( o, getReferencePointVec )
        if 'setRawData' in d:
            addMethod( o, setRawDataVec )
        if 'setReferencePoint' in d:
            addMethod( o, setReferencePointVec )
        if 'getVertex' in d:
            addMethod( o, getVertexVec )
        if 'setVertex' in d:
            addMethod( o, setVertexVec )
        if 'getCollection' in d:
            addMethod( o, getMcParticles )
            addMethod( o, getTracks )
        if 'getNumberOfElements' and 'getElementAt' in d:
            addMethod( o, LCCollectionIterator, '__iter__' )
        if 'getCollectionNames' and 'getCollection' in d:
            addMethod( o, LCEventIterator, '__iter__' )
        if 'readNextEvent' in d:
            addMethod( o, LCReaderIterator, '__iter__' )
        if 'updateNextEvent' in d:
            addMethod( o, LCStdHepRdrIterator, '__iter__' )
        decoratedClasses.append( o )

def decorateLcioClasses():
    ''' Standard method to decorate all LCIO classes '''
    for lcioClass in lcioClasses:
        decorateClass( lcioClass )
<commit_msg>replace usage of func_name with __name__<commit_after>'''
Created on Dec 4, 2012

Module to handle dynamic decoration of LCIO classes

@author: Christian Grefe
'''

from __future__ import absolute_import, unicode_literals
from ROOT import TVector3, TLorentzVector
from pyLCIO import EVENT, IMPL, IO, UTIL
from array import array

def addMethod( self, method, name=None ):
    ''' Convenience method to add a method to a class '''
    if not name:
        name = method.__name__
    setattr( self, name, method )

def getEndpointVec( self ):
    ''' Wrapper for the getEndpoint method to return a TVector3 '''
    v = self.getEndpoint()
    return TVector3( v[0], v[1], v[2] )

def getLorentzVec( self ):
    ''' Convenience method for classes that have momentum and energy '''
    energy = self.getEnergy()
    v = self.getMomentum()
    return TLorentzVector( v[0], v[1], v[2], energy )

def getMomentumVec( self ):
    ''' Wrapper for the getMomentum method to return a TVector3 '''
    v = self.getMomentum()
    return TVector3( v[0], v[1], v[2] )

def setMomentumVec( self, v ):
    ''' Wrapper for the setMomentum method passing a vector '''
    self.setMomentum( array( 'f', [v.x(), v.y(), v.z()] ) )

def getPositionVec( self ):
    ''' Wrapper for the getPosition method to return a TVector3 '''
    v = self.getPosition()
    return TVector3( v[0], v[1], v[2] )

def setPositionVec( self, v ):
    ''' Wrapper for the setPosition method passing a vector '''
    # tracker hits take pointer to doubles, calorimeter hits pointer to floats
    try:
        self.setPosition( array( 'd', [v.x(), v.y(), v.z()] ) )
    except TypeError:
        self.setPosition( array( 'f', [v.x(), v.y(), v.z()] ) )

def setRawDataVec( self, v ):
    ''' Wrapper for the setRawData method passing a vector '''
    self.setRawData( array( 'i', v ), len(v) )

def getReferencePointVec( self ):
    ''' Wrapper for the getReferencePoint method to return a TVector3 '''
    v = self.getReferencePoint()
    return TVector3( v[0], v[1], v[2] )

def setReferencePointVec( self, v ):
    ''' Wrapper for the setReferencePoint method passing a vector '''
    self.setReferencePoint( array( 'f', [v.x(), v.y(), v.z()] ) )

def getVertexVec( self ):
    ''' Wrapper for the getVertex method to return a TVector3 '''
    v = self.getVertex()
    return TVector3( v[0], v[1], v[2] )

def setVertexVec( self, v ):
    ''' Wrapper for the setVertex method passing a vector '''
    self.setVertex( array( 'd', [v.x(), v.y(), v.z()] ) )

def LCCollectionIterator( self ):
    ''' Helper method to make LCCollection iterable '''
    index = 0
    nElements = self.getNumberOfElements()
    while index < nElements:
        yield self.getElementAt( index )
        index += 1

def LCEventIterator( self ):
    ''' Helper method to make LCEvent iterable '''
    for collectionName in self.getCollectionNames():
        yield collectionName, self.getCollection( collectionName )

def LCReaderIterator( self ):
    ''' Helper method to make LCReader iterable '''
    event = self.readNextEvent()
    while event:
        yield event
        event = self.readNextEvent()

def LCStdHepRdrIterator( self ):
    ''' Helper method to make LCStdHepRdr iterable '''
    processedEvents = 0
    while True:
        event = IMPL.LCEventImpl()
        try:
            self.updateNextEvent( event )
            event.setEventNumber( processedEvents )
        except Exception:
            break
        yield event
        processedEvents += 1

def getMcParticles( self ):
    ''' Convenience method to get the default mc particle collection '''
    return self.getCollection( EVENT.LCIO.MCPARTICLE )

def getTracks( self ):
    ''' Convenience method to get the default track collection '''
    return self.getCollection( EVENT.LCIO.TRACK )

# List of LCIO classes to decorate
lcioClasses = [ EVENT.LCEvent,
                EVENT.CalorimeterHit,
                EVENT.Cluster,
                EVENT.RawCalorimeterHit,
                EVENT.ReconstructedParticle,
                EVENT.SimCalorimeterHit,
                EVENT.SimTrackerHit,
                EVENT.TPCHit,
                EVENT.Track,
                EVENT.TrackerData,
                EVENT.TrackerHit,
                EVENT.TrackState,
                EVENT.Vertex,
                EVENT.MCParticle,
                EVENT.LCCollection,
                IMPL.MCParticleImpl,
                IMPL.CalorimeterHitImpl,
                IMPL.ClusterImpl,
                IMPL.RawCalorimeterHitImpl,
                IMPL.ReconstructedParticleImpl,
                IMPL.SimCalorimeterHitImpl,
                IMPL.SimTrackerHitImpl,
                IMPL.TPCHitImpl,
                IMPL.TrackerDataImpl,
                IMPL.TrackerHitImpl,
                IMPL.TrackerHitPlaneImpl,
                IMPL.TrackerHitZCylinderImpl,
                IMPL.TrackerPulseImpl,
                IMPL.TrackerRawDataImpl,
                IMPL.TrackImpl,
                IMPL.TrackStateImpl,
                IMPL.VertexImpl,
                IO.LCReader,
                UTIL.LCStdHepRdr,
                ]

# Cache decorated classes
decoratedClasses = []

def decorateClass( o ):
    ''' Method to wrap the decoration logic '''
    if o not in decoratedClasses:
        d = dir( o )
        if 'getEndpoint' in d:
            addMethod( o, getEndpointVec )
        if 'getMomentum' in d:
            addMethod( o, getMomentumVec )
        if 'setMomentum' in d:
            addMethod( o, setMomentumVec )
        if 'getMomentum' and 'getEnergy' in d:
            addMethod( o, getLorentzVec )
        if 'getPosition' in d:
            addMethod( o, getPositionVec )
        if 'setPosition' in d:
            addMethod( o, setPositionVec )
        if 'getReferencePoint' in d:
            addMethod( o, getReferencePointVec )
        if 'setRawData' in d:
            addMethod( o, setRawDataVec )
        if 'setReferencePoint' in d:
            addMethod( o, setReferencePointVec )
        if 'getVertex' in d:
            addMethod( o, getVertexVec )
        if 'setVertex' in d:
            addMethod( o, setVertexVec )
        if 'getCollection' in d:
            addMethod( o, getMcParticles )
            addMethod( o, getTracks )
        if 'getNumberOfElements' and 'getElementAt' in d:
            addMethod( o, LCCollectionIterator, '__iter__' )
        if 'getCollectionNames' and 'getCollection' in d:
            addMethod( o, LCEventIterator, '__iter__' )
        if 'readNextEvent' in d:
            addMethod( o, LCReaderIterator, '__iter__' )
        if 'updateNextEvent' in d:
            addMethod( o, LCStdHepRdrIterator, '__iter__' )
        decoratedClasses.append( o )

def decorateLcioClasses():
    ''' Standard method to decorate all LCIO classes '''
    for lcioClass in lcioClasses:
        decorateClass( lcioClass )
<|endoftext|>


<commit_before>import logging
from conversation.models import (
    Tweetdj,
    PostgresqlDictionary,
    TwitterLanguageIdentifier,
)
from moderation.tasks import handle_socialuser_language
from conversation.tasks import handle_text_search_vector
from django.contrib.postgres.search import SearchVector
from django.db.models import Value
from moderation.models import addsocialuser
from whatthelang import WhatTheLang

logger = logging.getLogger(__name__)

class TextSearchVector():

    ok_lang = ["en", "fr"]
    all_entities = ["media", "urls", "user_mentions", "symbols", "hashtags"]
    entities = ["media", "urls", "user_mentions", "symbols"]
    wtl = WhatTheLang()
    try:
        simple_dict = PostgresqlDictionary.objects.get(cfgname="simple")
    except PostgresqlDictionary.DoesNotExist:
        simple_dict = None

    def __init__(self, tweetdj: Tweetdj):
        self.tweetdj = tweetdj
        self.text = self.get_text()
        self.dictionary: PostgresqlDictionary | None = None

    def get_text(self):
        try:
            return self.tweetdj.json["full_text"]
        except TypeError:
            return
        except KeyError:
            try:
                return self.tweetdj.json["text"]
            except KeyError:
                logger.error(
                    f"status {self.tweetdj.statusid}:\n"
                    "json field contains neither 'full_text' nor 'text' key"
                )

    def is_entity_only(self):
        count = 0
        for entity_key in TextSearchVector.all_entities:
            try:
                entity_lst = self.tweetdj.json["entities"][entity_key]
            except KeyError:
                continue
            count += len(entity_lst)
        return len(self.get_text().split()) == count

    def expunge_entities(self):
        # get list of indices tuples
        indices: list(tuple) = []
        for entity_key in TextSearchVector.entities:
            try:
                entity_lst = self.tweetdj.json["entities"][entity_key]
            except KeyError:
                continue
            for entity in entity_lst:
                indices.append(tuple(entity["indices"]))
        if not indices:
            return
        indices.sort(key=lambda x: x[0])
        new = ""
        txt = self.text
        for i,j in indices:
            new += txt[:i]
            new += " "*(j-i)
            new += txt[j:]
            txt=new
            new=""
        self.text=txt

    def remove_hash(self):
        # remove hash from hashtags
        indices: list(tuple) = []
        try:
            entity_lst = self.tweetdj.json["entities"]["hashtags"]
        except KeyError:
            return
        for entity in entity_lst:
            indices.append(tuple(entity["indices"]))
        if not indices:
            return
        indices.sort(key=lambda x: x[0])
        new = ""
        txt = self.text
        for i, _ in indices:
            new += txt[:i]
            new += " "
            new += txt[i+1:]
            txt=new
            new=""
        self.text=txt

    def remove_invalid_screen_name(self):
        """ Remove words starting with '@' corresponding to defunct
        screen_names """
        self.text=' '.join(
            [word for word in self.text.split() if word[0] != "@"]
        )

    def get_lang_socialuser(self):
        try:
            return self.tweetdj.socialuser.language.tag
        except (AttributeError, TypeError):
            addsocialuser(self.tweetdj)
            self.tweetdj.refresh_from_db()
            self.tweetdj.socialuser.refresh_from_db()
            try:
                return self.tweetdj.socialuser.language.tag
            except:
                ##handle_socialuser_language.si(self.tweetdj.socialuser.id)
                ##| handle_text_search_vector.si(self.tweetdj.statusid)
                handle_socialuser_language(self.tweetdj.socialuser.id)
                return self.get_lang_socialuser()

    def get_lang(self):
        try:
            tag = self.tweetdj.json["lang"]
            logger.debug(f'\ntwitter {tag=}')
        except (TypeError, KeyError) as e:
            logger.debug(f'{self.tweetdj}: {e}')
            return
        if tag in TextSearchVector.ok_lang:
            return tag
        try:
            tag = TextSearchVector.wtl.predict_lang(self.text)
        except ValueError as e:
            logger.error(e)
            tag = None
        logger.debug(f'\nwtl{tag=}')
        if tag in TextSearchVector.ok_lang:
            return tag
        tag = self.get_lang_socialuser()
        logger.debug(f'\nsocialuser {tag=}')
        if tag in TextSearchVector.ok_lang:
            return tag

    def get_dictionary(self, tag):
        try:
            dict = (
                TwitterLanguageIdentifier.objects.get(tag=tag)
                .postgresql_dictionary
            )
        except TwitterLanguageIdentifier.DoesNotExist as e:
            logger.error(f'{self.tweetdj}: {tag=} {e}')
            dict = None
        return dict or TextSearchVector.simple_dict

    def update_status_text(self):
        if not self.tweetdj.json:
            logger.debug("\njson is None")
            return
        if self.is_entity_only():
            logger.debug("\nonly entities")
            return
        self.expunge_entities()
        self.remove_hash()
        self.remove_invalid_screen_name()
        if not self.text:
            logger.debug("\nno text lef")
            return
        tag = self.get_lang()
        if tag:
            self.dictionary = self.get_dictionary(tag)
        else:
            logger.debug("no allowed lang tag found: skip")
            return
        sv=SearchVector(
            Value(self.text),
            config=self.dictionary.cfgname
        )
        self.tweetdj.status_text=sv
        self.tweetdj.save()
        #logger.debug(f'{self.tweetdj.statusid}: {self.tweetdj.status_text}')
<commit_msg>fix recursion in get_lang_socialuser method<commit_after>import logging
from conversation.models import (
    Tweetdj,
    PostgresqlDictionary,
    TwitterLanguageIdentifier,
)
from moderation.tasks import handle_socialuser_language
from conversation.tasks import handle_text_search_vector
from django.contrib.postgres.search import SearchVector
from django.db.models import Value
from moderation.models import addsocialuser
from whatthelang import WhatTheLang

logger = logging.getLogger(__name__)

class TextSearchVector():

    ok_lang = ["en", "fr"]
    all_entities = ["media", "urls", "user_mentions", "symbols", "hashtags"]
    entities = ["media", "urls", "user_mentions", "symbols"]
    wtl = WhatTheLang()
    try:
        simple_dict = PostgresqlDictionary.objects.get(cfgname="simple")
    except PostgresqlDictionary.DoesNotExist:
        simple_dict = None

    def __init__(self, tweetdj: Tweetdj):
        self.tweetdj = tweetdj
        self.text = self.get_text()
        self.dictionary: PostgresqlDictionary | None = None

    def get_text(self):
        try:
            return self.tweetdj.json["full_text"]
        except TypeError:
            return
        except KeyError:
            try:
                return self.tweetdj.json["text"]
            except KeyError:
                logger.error(
                    f"status {self.tweetdj.statusid}:\n"
                    "json field contains neither 'full_text' nor 'text' key"
                )

    def is_entity_only(self):
        count = 0
        for entity_key in TextSearchVector.all_entities:
            try:
                entity_lst = self.tweetdj.json["entities"][entity_key]
            except KeyError:
                continue
            count += len(entity_lst)
        return len(self.get_text().split()) == count

    def expunge_entities(self):
        # get list of indices tuples
        indices: list(tuple) = []
        for entity_key in TextSearchVector.entities:
            try:
                entity_lst = self.tweetdj.json["entities"][entity_key]
            except KeyError:
                continue
            for entity in entity_lst:
                indices.append(tuple(entity["indices"]))
        if not indices:
            return
        indices.sort(key=lambda x: x[0])
        new = ""
        txt = self.text
        for i,j in indices:
            new += txt[:i]
            new += " "*(j-i)
            new += txt[j:]
            txt=new
            new=""
        self.text=txt

    def remove_hash(self):
        # remove hash from hashtags
        indices: list(tuple) = []
        try:
            entity_lst = self.tweetdj.json["entities"]["hashtags"]
        except KeyError:
            return
        for entity in entity_lst:
            indices.append(tuple(entity["indices"]))
        if not indices:
            return
        indices.sort(key=lambda x: x[0])
        new = ""
        txt = self.text
        for i, _ in indices:
            new += txt[:i]
            new += " "
            new += txt[i+1:]
            txt=new
            new=""
        self.text=txt

    def remove_invalid_screen_name(self):
        """ Remove words starting with '@' corresponding to defunct
        screen_names """
        self.text=' '.join(
            [word for word in self.text.split() if word[0] != "@"]
        )

    def get_lang_socialuser(self):
        try:
            return self.tweetdj.socialuser.language.tag
        except AttributeError:
            addsocialuser(self.tweetdj)
            self.tweetdj.refresh_from_db()
            try:
                return self.tweetdj.socialuser.language.tag
            except AttributeError:
                handle_socialuser_language(self.tweetdj.socialuser.id)
                su = self.tweetdj.socialuser
                su.refresh_from_db()
                try:
                    return su.language.tag
                except:
                    return

    def get_lang(self):
        try:
            tag = self.tweetdj.json["lang"]
            logger.debug(f'\ntwitter {tag=}')
        except (TypeError, KeyError) as e:
            logger.debug(f'{self.tweetdj}: {e}')
            return
        if tag in TextSearchVector.ok_lang:
            return tag
        try:
            tag = TextSearchVector.wtl.predict_lang(self.text)
        except ValueError as e:
            logger.error(e)
            tag = None
        logger.debug(f'\nwtl{tag=}')
        if tag in TextSearchVector.ok_lang:
            return tag
        tag = self.get_lang_socialuser()
        logger.debug(f'\nsocialuser {tag=}')
        if tag in TextSearchVector.ok_lang:
            return tag

    def get_dictionary(self, tag):
        try:
            dict = (
                TwitterLanguageIdentifier.objects.get(tag=tag)
                .postgresql_dictionary
            )
        except TwitterLanguageIdentifier.DoesNotExist as e:
            logger.error(f'{self.tweetdj}: {tag=} {e}')
            dict = None
        return dict or TextSearchVector.simple_dict

    def update_status_text(self):
        if not self.tweetdj.json:
            logger.debug("\njson is None")
            return
        if self.is_entity_only():
            logger.debug("\nonly entities")
            return
        self.expunge_entities()
        self.remove_hash()
        self.remove_invalid_screen_name()
        if not self.text:
            logger.debug("\nno text lef")
            return
        tag = self.get_lang()
        if tag:
            self.dictionary = self.get_dictionary(tag)
        else:
            logger.debug("no allowed lang tag found: skip")
            return
        sv=SearchVector(
            Value(self.text),
            config=self.dictionary.cfgname
        )
        self.tweetdj.status_text=sv
        self.tweetdj.save()<|endoftext|>


<commit_before>__copyright__ = "Copyright 2017 Birkbeck, University of London"
__author__ = "Martin Paul Eve & Andy Byers"
__license__ = "AGPL v3"
__maintainer__ = "Birkbeck Centre for Technology and Publishing"

from django.urls import reverse
from django.shortcuts import redirect, render
from django.contrib import messages

from utils import setting_handler, models
from security.decorators import editor_user_required

@editor_user_required
def html_settings(request):
    plugin = models.Plugin.objects.get(name='HTML')

    try:
        html_block_content = setting_handler.get_plugin_setting(
            plugin,
            'html_block_content',
            request.journal,
            create=False,
            pretty='HTML Block Content'
        ).value
    except models.PluginSetting.DoesNotExist:
        messages.add_message(
            request,
            messages.ERROR,
            "No 'html_block_content' setting found. Have you installed "
            "this plugin?",
        )
        return redirect(
            reverse(
                'home_settings_index',
            )
        )

    if request.POST:
        html_block_content = request.POST.get('html_block_content')
        setting_handler.save_plugin_setting(plugin,
                                            'html_block_content',
                                            html_block_content,
                                            request.journal)
        messages.add_message(request, messages.INFO, 'HTML Block updated.')
        return redirect(reverse('home_settings_index'))

    template = 'html_settings.html'
    context = {
        'html_block_content': html_block_content,
        'disable_rich_text': request.GET.get('disable_rich_text', False)
    }

    return render(request, template, context)
<commit_msg>1111: HTML Plugin to create setting value<commit_after>__copyright__ = "Copyright 2017 Birkbeck, University of London"
__author__ = "Martin Paul Eve & Andy Byers"
__license__ = "AGPL v3"
__maintainer__ = "Birkbeck Centre for Technology and Publishing"

from django.urls import reverse
from django.shortcuts import redirect, render
from django.contrib import messages

from utils import setting_handler, models
from security.decorators import editor_user_required

@editor_user_required
def html_settings(request):
    plugin = models.Plugin.objects.get(name='HTML')

    html_block_content = setting_handler.get_plugin_setting(
        plugin,
        'html_block_content',
        request.journal,
        create=True,
        pretty='HTML Block Content'
    ).value

    if request.POST:
        html_block_content = request.POST.get('html_block_content')
        setting_handler.save_plugin_setting(plugin,
                                            'html_block_content',
                                            html_block_content,
                                            request.journal)
        messages.add_message(request, messages.INFO, 'HTML Block updated.')
        return redirect(reverse('home_settings_index'))

    template = 'html_settings.html'
    context = {
        'html_block_content': html_block_content,
        'disable_rich_text': request.GET.get('disable_rich_text', False)
    }

    return render(request, template, context)
<|endoftext|>


<commit_before>"""
The worker function in this module performs the order processing.
"""
import logging
import simplejson
from src.core.market_data import SerializableOrderList
from src.daemons.gateway import parsers

logger = logging.getLogger(__name__)

def parser_order(job_json):
    """
    Routes the order to the correct parser, returns the parsed order in
    Unified Uploader format.

    :param str job_json: A raw JSON job dict string to parse.
    :rtype: SerializableOrderList
    :returns: A serializable list of MarketOrder objects.
    """
    job_dict = simplejson.loads(job_json)
    # Orders are lumped into this list sub-class, which has JSON-serialization
    # methods on it.
    order_list = SerializableOrderList()

    # The format attrib on the job dict determines which parser to use.
    order_format = job_dict.get('format', 'unknown')

    try:
        # A payload must exist, regardless of the format. The payload contains
        # the data passed to the gateway.
        payload = job_dict['payload']
    except KeyError:
        logger.error('Job dict has no payload key. Discarding.')
        return

    if order_format == 'eve_marketeer':
        order_generator = parsers.eve_marketeer.parse_from_payload(payload)
    else:
        logger.error('Unknown order format encountered. Discarding.')
        return

    # This generator spits out the MarketOrder instances that user sent. Since
    # one item can have many orders, we add each order entry to our
    # SerializableOrderList instance.
    for order in order_generator:
        order_list.append(order)

    return order_list

def worker(job_json, sender):
    """
    TODO: Make this useful.
    """
    logger.info('Processing one job.')

    order_list = parser_order(job_json)
    if order_list:
        sender.send(order_list.to_json())
<commit_msg>Processor comments.<commit_after>"""
The worker function in this module performs the order processing.
"""
import logging
import simplejson
from src.core.market_data import SerializableOrderList
from src.daemons.gateway import parsers

logger = logging.getLogger(__name__)

def parser_order(job_json):
    """
    Routes the order to the correct parser, returns the parsed order in
    Unified Uploader format.

    :param str job_json: A raw JSON job dict string to parse.
    :rtype: SerializableOrderList
    :returns: A serializable list of MarketOrder objects.
    """
    job_dict = simplejson.loads(job_json)
    # Orders are lumped into this list sub-class, which has JSON-serialization
    # methods on it.
    order_list = SerializableOrderList()

    # The format attrib on the job dict determines which parser to use.
    order_format = job_dict.get('format', 'unknown')

    try:
        # A payload must exist, regardless of the format. The payload contains
        # the data passed to the gateway.
        payload = job_dict['payload']
    except KeyError:
        logger.error('Job dict has no payload key. Discarding.')
        return

    if order_format == 'eve_marketeer':
        order_generator = parsers.eve_marketeer.parse_from_payload(payload)
    else:
        logger.error('Unknown order format encountered. Discarding.')
        return

    # This generator spits out the MarketOrder instances that user sent. Since
    # one item can have many orders, we add each order entry to our
    # SerializableOrderList instance.
    for order in order_generator:
        order_list.append(order)

    return order_list

def worker(job_json, sender):
    """
    Parses the job dict into a SerializableOrderList of MarketOrder instances.
    This gets dumped to JSON and sent on to the relays for sending to the
    subscribers.

    :param str job_json: The job dict from the broker/gateway.
    :param zmq.socket: The socket to send orders to the relays with.
    """
    logger.info('Processing one job.')

    order_list = parser_order(job_json)
    if order_list:
        sender.send(order_list.to_json())
<|endoftext|>


<commit_before>import uuid
import signal
import logging

import amqp

import gevent
import gevent.monkey

# Exchange to be used by hub. Should be the same for server and clients.
EXCHANGE = 'hub'

# Routing queue/key for hub requests from clients to servers.
# Should be the same for server and clients.
REQUESTS_KEY = 'hub'

gevent.monkey.patch_all()

log = logging.getLogger(__name__)

class AmqpHelper(object):
    def __init__(self):
        self.conns = {}
        self.chans = {}
        self.lbl = self.__class__.__name__

    @property
    def greenlet_id(self):
        """Find current greenlet's id"""
        return id(gevent.getcurrent())

    @property
    def conn(self):
        """Find or create current greenlet's amqp connection"""
        gid = self.greenlet_id
        if gid not in self.conns:
            log.debug("%s: Opening new amqp connection.", self.lbl)
            self.conns[gid] = amqp.Connection()
        return self.conns[gid]

    def close_conn(self):
        """Close current greenlet's amqp connection"""
        gid = self.greenlet_id
        if gid not in self.conns:
            log.warning("%s: No connection open to close.", self.lbl)
        else:
            self.conns.pop(gid).close()

    @property
    def chan(self):
        """Find or create current greenlet's amqp channel"""
        gid = self.greenlet_id
        if gid not in self.chans:
            conn = self.conn
            log.debug("%s: Opening new amqp channel.", self.lbl)
            self.chans[gid] = conn.channel()
        return self.chans[gid]

    def close_chan(self):
        """Close current greenlet's amqp channel"""
        gid = self.greenlet_id
        if gid not in self.chans:
            log.warning("%s: No channel open to close.", self.lbl)
        else:
            self.chans.pop(gid).close()

    def listen_amqp(self):
        """Block on channel messages until an exception raises"""
        log.info("%s (AmqpHelper): Starting amqp subscriber.", self.lbl)
        try:
            while True:
                self.chan.wait()
        except BaseException as exc:
            log.error("%s: Amqp consumer received %r, stopping.",
                      self.lbl, exc)
            self.close_chan()

    def stop(self):
        for gid in self.chans.keys():
            self.chans.pop(gid).close()
        while self.conns.keys():
            self.conns.pop(gid).close()

class Server(object):
    """Hub Server"""

    def __init__(self, exchange=EXCHANGE, key=REQUESTS_KEY, workers=None):
        """Initialize a Hub Server"""

        self.exchange = exchange
        self.key = key
        self.worker_cls = {'default': Worker}
        self.worker_cls.update(workers or {})
        self.workers = {}
        self.listener = None
        self.stopped = False
        self.lbl = "Hub Server"
        log.info("%s: Initializing.", self.lbl)

        # initialize amqp connection and channel, declare exchange
        self.conn = amqp.Connection()
        self.chan = self.conn.channel()
        self.chan.exchange_declare(self.exchange, 'topic')
        log.info("%s: Will use exchange '%s'.", self.lbl, self.exchange)

        # declare, bind, set consumer for rpc calls
        self.chan.basic_qos(0, 1, False)  # prefetch count = 1
        self.chan.queue_declare(self.key, exclusive=True)
        self.chan.queue_bind(self.key, self.exchange, '%s.#' % self.key)
        self.chan.basic_consume(self.key, callback=self.on_rpc)
        log.info("%s: RPC queue '%s' with routing key '%s.#'.",
                 self.lbl, self.key, self.key)

        log.info("%s: Initialized.", self.lbl)

    def on_rpc(self, msg):
        """Handle amqp rpc messages"""
        routing_key = msg.delivery_info.get('routing_key', '')
        log.info("%s: Received rpc message with routing_key '%s'.",
                 self.lbl, routing_key)
        try:
            route_parts = routing_key.split('.')
            assert len(route_parts) == 2
            assert route_parts[0] == self.key
            assert route_parts[1] in self.worker_cls
        except AssertionError:
            log.error("%s: Invalid routing key '%s'.", self.lbl, routing_key)
            return
        try:
            assert 'correlation_id' in msg.properties
            assert 'reply_to' in msg.properties
            correlation_id = msg.properties['correlation_id']
            reply_to = msg.properties['reply_to']
        except AssertionError:
            log.error("%s: Couldn't find correlation_id and reply_to for "
                      "response. Properties: %s", self.lbl, msg.properties)
            return
        log.info("%s: Msg has correlation_id '%s' and reply_to '%s'.",
                 self.lbl, correlation_id, reply_to)
        worker_cls = self.worker_cls[route_parts[1]]
        worker = worker_cls(msg.body, self.exchange)
        self.workers[worker.uuid] = worker
        worker.start()
        # send back rpc response
        msg = amqp.Message(worker.uuid, correlation_id=correlation_id)
        self.chan.basic_publish(msg, self.exchange, reply_to)

    def run(self):
        """Run the Hub Server"""
        if self.listener is not None:
            log.warning("%s: Can't call run() twice.", self.lbl)
            return
        log.info("%s: Starting.", self.lbl)
        self.listener = gevent.spawn(self.listen)
        self.listener.join()

    def listen(self):
        """Block on channel messages until an exception raises"""
        log.info("%s: Starting amqp consumer.", self.lbl)
        try:
            while True:
                self.chan.wait()
        except BaseException as exc:
            log.error("%s: Amqp consumer received %r, stopping.",
                      self.lbl, exc)

    def stop(self):
        """Stop Hub Server, kill workers, close connections"""
        if self.stopped:
            return
        log.info("%s: Stopping.", self.lbl)
        if self.listener is not None:
            self.listener.kill()
        try:
            self.chan.close()
            self.conn.close()
        except Exception as exc:
            log.warning("%s: Error stopping %r", self.lbl, exc)
        self.closed = True

class Worker(AmqpHelper):
    """Hub Worker"""

    def __init__(self, params, exchange=EXCHANGE):
        """Initialize a worker proxying a connection"""
        super(Worker, self).__init__()
        self.uuid = uuid.uuid4().hex
        self.lbl = "Hub Worker %s" % self.uuid
        log.info("%s: Initializing.", self.lbl)

        self.exchange = exchange
        self.greenlets = {}
        self.stopped = False

        log.info("%s: Initialized.", self.lbl)

    def start(self):
        """Start all greenlets for this worker"""
        if self.greenlets:
            log.warning("%s: Can't call start() twice.", self.lbl)
            return
        log.info("%s: Starting.", self.lbl)
        self.greenlets['amqp_listener'] = gevent.spawn(self.listen_amqp)

    def listen_amqp(self):
        """Block on channel messages until an exception raises"""
        log.info("%s: Starting amqp subscriber.""", self.lbl)
        self.chan.queue_declare(self.uuid, exclusive=True)
        self.chan.queue_bind(self.uuid, self.exchange, '%s.#' % self.uuid)
        self.chan.basic_consume(self.uuid, callback=self.handle_amqp_msg)
        log.info("%s: Exchange '%s', queue '%s' with routing key '%s.#'.",
                 self.lbl, self.exchange, self.uuid, self.uuid)
        super(Worker, self).listen_amqp()

    def handle_amqp_msg(self, msg):
        """Handle an incoming message from amqp

        A message's body with routing key <uuid>.action will be forwarded
        to self.on_<action> with the message's body as its single argument.

        """
        body = msg.body
        routing_key = msg.delivery_info.get('routing_key', '')
        log.info("%s: Received message with routing key '%s' and body %r.",
                 self.lbl, routing_key, body)
        try:
            route_parts = routing_key.split('.')
            assert len(route_parts) == 2, "Routing key should have 2 parts."
            assert route_parts[0] == self.uuid, "Worker uuid mismatch."
            action = route_parts[1]
            attr = 'on_%s' % action
            assert hasattr(self, attr), "Invalid action %s." % action
        except AssertionError as exc:
            log.error("%s: Error parsing amqp msg with routing key '%s' and "
                      "body %r. %s", self.lbl, routing_key, body, exc)
            return
        try:
            getattr(self, attr)(msg.body)
        except Exception as exc:
            log.error("%s: Error parsing amqp msg with routing key '%s' and "
                      "body %r. %s", self.lbl, routing_key, body, exc)

    def send_amqp_msg(self, data):
        """Send message to amqp"""
        log.info("%s: Sending message to amqp with body %r.", self.lbl, data)
        msg = data if isinstance(data, amqp.Message) else amqp.Message(data)
        self.chan.basic_publish(msg, self.exchange, self.uuid)

    def on_echo(self, data):
        """Echo message handler"""
        log.info("%s: Received on_echo %r. Will echo back.", self.lbl, data)
        self.send_amqp_msg(data)

    def stop(self):
        """Stop this worker"""
        if self.stopped:
            return
        log.info("%s: Stopping.", self.lbl)
        gevent.killall(self.greenlets.values())
        super(Worker, self).stop()
        self.closed = True

class Client(AmqpHelper):
    def __init__(self, exchange=EXCHANGE, key=REQUESTS_KEY,
                 worker_type='default', **worker_kwargs):
        super(Client, self).__init__()
        self.lbl = "Hub Client"
        log.info("%s: Initializing.", self.lbl)
        self.exchange = exchange
        self.key = key

        # define callback queue
        self.queue = self.chan.queue_declare(exclusive=True).queue
        self.chan.queue_bind(self.queue, self.exchange, self.queue)
        self.chan.basic_consume(self.queue, callback=self.on_msg)

        # send rpc request
        self.routing_key = None
        self.correlation_id = uuid.uuid4().hex
        reply_to = self.queue
        routing_key = '%s.%s' % (self.key, worker_type)
        msg = amqp.Message(worker_kwargs, correlation_id=self.correlation_id,
                           reply_to=reply_to)
        self.chan.basic_publish(msg, self.exchange, routing_key)

        # wait for rpc response
        try:
            while not self.routing_key:
                self.chan.wait()
        except BaseException as exc:
            log.error("%s: Amqp consumer received %r while waiting for RPC "
                      "response. Stopping.", self.lbl, exc)
        log.info("%s: Finished waiting for RPC response.", self.lbl)

    def on_msg(self, msg):
        body = msg.body
        routing_key = msg.delivery_info.get('routing_key', '')
        log.info("%s: Received message with routing key '%s' and body %r.",
                 self.lbl, routing_key, body)

        if not self.routing_key:
            # waiting for RPC response
            if not routing_key == self.queue:
                log.warning("%s: Got msg with routing key '%s' when expecting "
                            "'%s'.", self.lbl, routing_key, self.queue)
                return
            if self.correlation_id != msg.properties.get('correlation_id'):
                log.warning(
                    "%s: Got msg with corr_id '%s' when expecting '%s'.",
                    self.lbl, msg.properties.get('correlation_id'),
                    self.correlation_id
                )
                return
            self.routing_key = msg.body
            log.info("%s: Received RPC response with body %r.",
                     self.lbl, msg.body)
        else:
            log.info("%s: Received msg with body %r.", self.lbl, msg.body)

def main():
    logfmt = "[%(asctime)-15s][%(levelname)s] %(module)s - %(message)s"
    loglvl = logging.DEBUG
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter(logfmt))
    handler.setLevel(loglvl)
    logging.root.addHandler(handler)
    logging.root.setLevel(loglvl)

    hub = Server()

    def sig_handler(sig=None, frame=None):
        log.warning("Hubo process received SIGTERM/SIGINT")
        hub.stop()

    # gevent.signal(signal.SIGTERM, sig_handler)
    # gevent.signal(signal.SIGINT, sig_handler)  # also catch KeyboardInterrupt

    try:
        hub.run()
    except BaseException as exc:
        log.error("Hub run interrupted by exception: %r", exc)
        hub.stop()

if __name__ == "__main__":
    main()
<commit_msg>hub 8: fix read/write queue communication<commit_after>import uuid
import signal
import logging

import amqp

import gevent
import gevent.monkey

# Exchange to be used by hub. Should be the same for server and clients.
EXCHANGE = 'hub'

# Routing queue/key for hub requests from clients to servers.
# Should be the same for server and clients.
REQUESTS_KEY = 'hub'

gevent.monkey.patch_all()

log = logging.getLogger(__name__)

class AmqpHelper(object):
    def __init__(self):
        self.conns = {}
        self.chans = {}
        self.lbl = self.__class__.__name__

    @property
    def greenlet_id(self):
        """Find current greenlet's id"""
        return id(gevent.getcurrent())

    @property
    def conn(self):
        """Find or create current greenlet's amqp connection"""
        gid = self.greenlet_id
        if gid not in self.conns:
            log.debug("%s: Opening new amqp connection.", self.lbl)
            self.conns[gid] = amqp.Connection()
        return self.conns[gid]

    def close_conn(self):
        """Close current greenlet's amqp connection"""
        gid = self.greenlet_id
        if gid not in self.conns:
            log.warning("%s: No connection open to close.", self.lbl)
        else:
            self.conns.pop(gid).close()

    @property
    def chan(self):
        """Find or create current greenlet's amqp channel"""
        gid = self.greenlet_id
        if gid not in self.chans:
            conn = self.conn
            log.debug("%s: Opening new amqp channel.", self.lbl)
            self.chans[gid] = conn.channel()
        return self.chans[gid]

    def close_chan(self):
        """Close current greenlet's amqp channel"""
        gid = self.greenlet_id
        if gid not in self.chans:
            log.warning("%s: No channel open to close.", self.lbl)
        else:
            self.chans.pop(gid).close()

    def listen_amqp(self):
        """Block on channel messages until an exception raises"""
        log.info("%s (AmqpHelper): Starting amqp subscriber.", self.lbl)
        try:
            while True:
                self.chan.wait()
        except BaseException as exc:
            log.error("%s: Amqp consumer received %r, stopping.",
                      self.lbl, exc)
            self.close_chan()

    def stop(self):
        for gid in self.chans.keys():
            self.chans.pop(gid).close()
        while self.conns.keys():
            self.conns.pop(gid).close()

class Server(object):
    """Hub Server"""

    def __init__(self, exchange=EXCHANGE, key=REQUESTS_KEY, workers=None):
        """Initialize a Hub Server"""

        self.exchange = exchange
        self.key = key
        self.worker_cls = {'default': Worker}
        self.worker_cls.update(workers or {})
        self.workers = {}
        self.listener = None
        self.stopped = False
        self.lbl = "Hub Server"
        log.info("%s: Initializing.", self.lbl)

        # initialize amqp connection and channel, declare exchange
        self.conn = amqp.Connection()
        self.chan = self.conn.channel()
        self.chan.exchange_declare(self.exchange, 'topic')
        log.info("%s: Will use exchange '%s'.", self.lbl, self.exchange)

        # declare, bind, set consumer for rpc calls
        self.chan.basic_qos(0, 1, False)  # prefetch count = 1
        self.chan.queue_declare(self.key, exclusive=True)
        self.chan.queue_bind(self.key, self.exchange, '%s.#' % self.key)
        self.chan.basic_consume(self.key, callback=self.on_rpc)
        log.info("%s: RPC queue '%s' with routing key '%s.#'.",
                 self.lbl, self.key, self.key)

        log.info("%s: Initialized.", self.lbl)

    def on_rpc(self, msg):
        """Handle amqp rpc messages"""
        routing_key = msg.delivery_info.get('routing_key', '')
        log.info("%s: Received rpc message with routing_key '%s'.",
                 self.lbl, routing_key)
        try:
            route_parts = routing_key.split('.')
            assert len(route_parts) == 2
            assert route_parts[0] == self.key
            assert route_parts[1] in self.worker_cls
        except AssertionError:
            log.error("%s: Invalid routing key '%s'.", self.lbl, routing_key)
            return
        try:
            assert 'correlation_id' in msg.properties
            assert 'reply_to' in msg.properties
            correlation_id = msg.properties['correlation_id']
            reply_to = msg.properties['reply_to']
        except AssertionError:
            log.error("%s: Couldn't find correlation_id and reply_to for "
                      "response. Properties: %s", self.lbl, msg.properties)
            return
        log.info("%s: Msg has correlation_id '%s' and reply_to '%s'.",
                 self.lbl, correlation_id, reply_to)
        worker_cls = self.worker_cls[route_parts[1]]
        worker = worker_cls(msg.body, self.exchange)
        self.workers[worker.uuid] = worker
        worker.start()
        # send back rpc response
        msg = amqp.Message(worker.uuid, correlation_id=correlation_id)
        self.chan.basic_publish(msg, self.exchange, reply_to)

    def run(self):
        """Run the Hub Server"""
        if self.listener is not None:
            log.warning("%s: Can't call run() twice.", self.lbl)
            return
        log.info("%s: Starting.", self.lbl)
        self.listener = gevent.spawn(self.listen)
        self.listener.join()

    def listen(self):
        """Block on channel messages until an exception raises"""
        log.info("%s: Starting amqp consumer.", self.lbl)
        try:
            while True:
                self.chan.wait()
        except BaseException as exc:
            log.error("%s: Amqp consumer received %r, stopping.",
                      self.lbl, exc)

    def stop(self):
        """Stop Hub Server, kill workers, close connections"""
        if self.stopped:
            return
        log.info("%s: Stopping.", self.lbl)
        if self.listener is not None:
            self.listener.kill()
        try:
            self.chan.close()
            self.conn.close()
        except Exception as exc:
            log.warning("%s: Error stopping %r", self.lbl, exc)
        self.closed = True

class Worker(AmqpHelper):
    """Hub Worker"""

    def __init__(self, params, exchange=EXCHANGE):
        """Initialize a worker proxying a connection"""
        super(Worker, self).__init__()
        self.uuid = uuid.uuid4().hex
        self.lbl = "Hub Worker %s" % self.uuid
        log.info("%s: Initializing.", self.lbl)

        self.exchange = exchange
        self.greenlets = {}
        self.stopped = False

        log.info("%s: Initialized.", self.lbl)

    def start(self):
        """Start all greenlets for this worker"""
        if self.greenlets:
            log.warning("%s: Can't call start() twice.", self.lbl)
            return
        log.info("%s: Starting.", self.lbl)
        self.greenlets['amqp_listener'] = gevent.spawn(self.listen_amqp)

    def listen_amqp(self):
        """Block on channel messages until an exception raises"""
        log.info("%s: Starting amqp subscriber.""", self.lbl)
        self.chan.queue_declare(self.uuid, exclusive=True)
        self.chan.queue_bind(self.uuid, self.exchange, '%s.#' % self.uuid)
        self.chan.basic_consume(self.uuid, callback=self.handle_amqp_msg)
        log.info("%s: Exchange '%s', queue '%s' with routing key '%s.#'.",
                 self.lbl, self.exchange, self.uuid, self.uuid)
        super(Worker, self).listen_amqp()

    def handle_amqp_msg(self, msg):
        """Handle an incoming message from amqp

        A message's body with routing key <uuid>.action will be forwarded
        to self.on_<action> with the message's body as its single argument.

        """
        body = msg.body
        routing_key = msg.delivery_info.get('routing_key', '')
        log.info("%s: Received message with routing key '%s' and body %r.",
                 self.lbl, routing_key, body)
        try:
            route_parts = routing_key.split('.')
            assert len(route_parts) == 2, "Routing key should have 2 parts."
            assert route_parts[0] == self.uuid, "Worker uuid mismatch."
            action = route_parts[1]
            attr = 'on_%s' % action
            assert hasattr(self, attr), "Invalid action %s." % action
        except AssertionError as exc:
            log.error("%s: Error parsing amqp msg with routing key '%s' and "
                      "body %r. %s", self.lbl, routing_key, body, exc)
            return
        try:
            getattr(self, attr)(msg.body)
        except Exception as exc:
            log.error("%s: Error parsing amqp msg with routing key '%s' and "
                      "body %r. %s", self.lbl, routing_key, body, exc)

    def send_amqp_msg(self, data):
        """Send message to amqp"""
        log.info("%s: Sending message to amqp with body %r.", self.lbl, data)
        msg = data if isinstance(data, amqp.Message) else amqp.Message(data)
        self.chan.basic_publish(msg, self.exchange, 'from_%s' % self.uuid)

    def on_echo(self, data):
        """Echo message handler"""
        log.info("%s: Received on_echo %r. Will echo back.", self.lbl, data)
        self.send_amqp_msg(data)

    def stop(self):
        """Stop this worker"""
        if self.stopped:
            return
        log.info("%s: Stopping.", self.lbl)
        gevent.killall(self.greenlets.values())
        super(Worker, self).stop()
        self.closed = True

class Client(AmqpHelper):
    def __init__(self, exchange=EXCHANGE, key=REQUESTS_KEY,
                 worker_type='default', **worker_kwargs):
        super(Client, self).__init__()
        self.lbl = "Hub Client"
        log.info("%s: Initializing.", self.lbl)
        self.exchange = exchange
        self.key = key

        # define callback queue
        self.queue = self.chan.queue_declare(exclusive=True).queue
        self.chan.queue_bind(self.queue, self.exchange, self.queue)
        self.chan.basic_consume(self.queue, callback=self.on_msg)

        # send rpc request
        self.worker_id = None
        self.correlation_id = uuid.uuid4().hex
        reply_to = self.queue
        routing_key = '%s.%s' % (self.key, worker_type)
        msg = amqp.Message(worker_kwargs, correlation_id=self.correlation_id,
                           reply_to=reply_to)
        self.chan.basic_publish(msg, self.exchange, routing_key)

        # wait for rpc response
        try:
            while not self.worker_id:
                self.chan.wait()
        except BaseException as exc:
            log.error("%s: Amqp consumer received %r while waiting for RPC "
                      "response. Stopping.", self.lbl, exc)
        log.info("%s: Finished waiting for RPC response.", self.lbl)

    def on_msg(self, msg):
        body = msg.body
        routing_key = msg.delivery_info.get('routing_key', '')
        log.info("%s: Received message with routing key '%s' and body %r.",
                 self.lbl, routing_key, body)

        if not self.worker_id:
            # waiting for RPC response
            if not routing_key == self.queue:
                log.warning("%s: Got msg with routing key '%s' when expecting "
                            "'%s'.", self.lbl, routing_key, self.queue)
                return
            if self.correlation_id != msg.properties.get('correlation_id'):
                log.warning(
                    "%s: Got msg with corr_id '%s' when expecting '%s'.",
                    self.lbl, msg.properties.get('correlation_id'),
                    self.correlation_id
                )
                return
            self.worker_id = msg.body
            log.info("%s: Received RPC response with body %r.",
                     self.lbl, msg.body)
            log.info("%s: Will start listening for routing_key 'from_%s'.",
                     self.lbl, self.worker_id)
            self.chan.queue_bind(self.queue, self.exchange,
                                 'from_%s' % self.worker_id)
        else:
            # receiving messages
            if not routing_key == 'from_%s' % self.worker_id:
                log.warning("%s: Got msg with routing key '%s' when expecting "
                            "'from_%s'.", self.lbl, routing_key,
                            self.worker_id)
                return
            log.info("%s: Received msg with body %r.", self.lbl, msg.body)

    def send_msg(self, action, msg):
        if not self.worker_id:
            raise Exception("Routing key not yet received in RPC response.")
        msg = msg if isinstance(msg, amqp.Message) else amqp.Message(msg)
        routing_key = '%s.%s' % (self.worker_id, action)
        log.info("%s: Will emit msg with body '%s' and routing_key '%s'.",
                 self.lbl, msg.body, routing_key)
        self.chan.basic_publish(msg, self.exchange, routing_key)

def main():
    logfmt = "[%(asctime)-15s][%(levelname)s] %(module)s - %(message)s"
    loglvl = logging.DEBUG
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter(logfmt))
    handler.setLevel(loglvl)
    logging.root.addHandler(handler)
    logging.root.setLevel(loglvl)

    hub = Server()

    def sig_handler(sig=None, frame=None):
        log.warning("Hubo process received SIGTERM/SIGINT")
        hub.stop()

    # gevent.signal(signal.SIGTERM, sig_handler)
    # gevent.signal(signal.SIGINT, sig_handler)  # also catch KeyboardInterrupt

    try:
        hub.run()
    except BaseException as exc:
        log.error("Hub run interrupted by exception: %r", exc)
        hub.stop()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># oh boy oh boy!
# It's project HOT_PANTS
from __future__ import print_function
import Adafruit_BBIO.UART as uart
import Adafruit_BBIO.ADC as adc
from Adafruit_Thermal import *
# import TMP102 as tmp
# import VCNL4000 as vcnl
import time
from serial import Serial
import random
import atexit
import sentence_generator as sg

theObj = 'APPLE'
sensor_pin = 'P9_40'
readings = []
fake = 0

def parseLen(text):
    L = []
    # add newlines to cause text to print properly
    # we need this because we're printing upside-down text
    # "call the police, it is faint-hearted" should be printed as
    # "rted" then linebreak then "Call the police, it is faint-hea"
    # which is "rted\nCall the police, it is faint-hea"
    # "faint-hearted\nCall the police, it is"

    if len(text) > Adafruit_Thermal.maxColumn: # 32 is defined by the printer; max chars per line
        r = len(text)%32
        L.append(text[-r:]+'\n')
        for i in reversed(range(len(text)/32)):
            L.append(text[i*32:(i+1)*32]+'\n')
    else:
        L.append(text)
    return ''.join(L)

def parse(text):
    r = text.split(' ')
    curLine = ''
    fin = []
    tally = 0
    for w in r:
        if len(w)+len(curLine) > (Adafruit_Thermal.maxColumn-1):
            fin.append(curLine)
            curLine = ''
            curLine+=w
        else:
            curLine+=' '+w
    # print curLine
    fin.append(curLine)
    fin[0] = fin[0].lstrip()
    fin.reverse()
    rt = '\n'.join(fin)
    return rt+'\n'

def slowPrint(text):
    for i in text.splitlines():
        printer.print(i+'\n')
        time.sleep(0.1)

def checkSensor():
    global rPast
    global rMin
    global rMax
    global noop
    # change this to whatever get-readings call we need
    r = adc.read(sensor_pin)
    readings.append(r)
    if len(readings)>WINDOW_SIZE:
        del readings[:-WINDOW_SIZE]

    avg = 0
    for i in readings:
        avg += (i/float(len(readings)))

    delta = r-rPast
    # delta = r-avg

    print('%0.3f\t%0.3f\t%0.3f\t%0.3f\t%0.3f'%(r, delta, avg, rMin, rMax))

    if r > rMax:
        rMax = r
        # does this merit an emission? Or should delta have to be > threshold?
    if r < rMin:
        rMin = r
        # does this merit an emission? Or should delta have to be > threshold?

    if abs(delta) > emission_threshold:
        if len(readings)==WINDOW_SIZE: # this test prevents emissions before WINDOW_SIZE samples have been taken
            # print('emitting remark')
            noop = 0
            emit_remark(r, delta, avg)
        else:
            pass
    else:
        noop += 1
        if noop > noop_threshold:
            noop = 0
            # print('emitting dream')
            emit_dream(r, delta, avg)
    rPast = r

def emit_dream(r, delta, avg):
    global fake
    if fake == 5:
        fake = 0
        norm = mapVals((1.0-r), rMin, rMax, 0.0, 0.999)
        sen = sg.generate(theObj, norm, delta, True)

        # printer.flush()
        printer.feed(1)
        printer.print('            . . .             ')
        printer.feed(1)

        slowPrint(parse(sen))

        # printer.flush()
        printer.feed(1)
        printer.print('            . . .             ')
        printer.feed(2)
    else:
        fake += 1
        norm = mapVals((1.0-r), rMin, rMax, 0.0, 0.999)
        sen = sg.generate(theObj, norm, delta, False)
        slowPrint(parse(sen))
        printer.feed(2)

def emit_remark(r, delta, avg):
    norm = mapVals((1.0-r), rMin, rMax, 1.0, 0.0)
    sen = sg.generate(theObj, norm, delta, False)
    slowPrint(parse(sen))
    # slowPrint(str(norm))
    printer.feed(2)

def exit_handler():
    print('exiting')
    # adc.cleanup()
    # uart.cleanup() # not yet supported?

def mapVals(val, inMin, inMax, outMin, outMax):
        toRet = float(outMin + (float(outMax - outMin) * (float(val - inMin) / float(inMax - inMin))))
        toRet = clamp(toRet, outMin, outMax)
        print('turned %s into %s'%(val,toRet))
        return toRet

def clamp(val, tmin, tmax):
    if val > tmax:
        val = tmax
    if val < tmin:
        val = tmin
    return val

uart.setup("UART2")
adc.setup()
atexit.register(exit_handler)
printer = Adafruit_Thermal("/dev/ttyO2", 19200, timeout=5)
printer.begin()
printer.upsideDownOn()
printer.feed(3)
printer.print(parse('i am awake and I am APPLE (light)'))
printer.feed(1)
rPast = 0
rMax = 0 # all-time max sensor reading
rMin = 100000 # all-time min sensor reading
WINDOW_SIZE = 30 # size of moving-window avg
noop = 0 # number of intervals passed without a trigger
noop_threshold = 480
emission_threshold = 0.01 # changed this for vcnl4000, used to be 0.7

while True:
    checkSensor()
    time.sleep(0.5)<commit_msg>reverse sensor range<commit_after># oh boy oh boy!
# It's project HOT_PANTS
from __future__ import print_function
import Adafruit_BBIO.UART as uart
import Adafruit_BBIO.ADC as adc
from Adafruit_Thermal import *
# import TMP102 as tmp
# import VCNL4000 as vcnl
import time
from serial import Serial
import random
import atexit
import sentence_generator as sg

theObj = 'APPLE'
sensor_pin = 'P9_40'
readings = []
fake = 0

def parseLen(text):
    L = []
    # add newlines to cause text to print properly
    # we need this because we're printing upside-down text
    # "call the police, it is faint-hearted" should be printed as
    # "rted" then linebreak then "Call the police, it is faint-hea"
    # which is "rted\nCall the police, it is faint-hea"
    # "faint-hearted\nCall the police, it is"

    if len(text) > Adafruit_Thermal.maxColumn: # 32 is defined by the printer; max chars per line
        r = len(text)%32
        L.append(text[-r:]+'\n')
        for i in reversed(range(len(text)/32)):
            L.append(text[i*32:(i+1)*32]+'\n')
    else:
        L.append(text)
    return ''.join(L)

def parse(text):
    r = text.split(' ')
    curLine = ''
    fin = []
    tally = 0
    for w in r:
        if len(w)+len(curLine) > (Adafruit_Thermal.maxColumn-1):
            fin.append(curLine)
            curLine = ''
            curLine+=w
        else:
            curLine+=' '+w
    # print curLine
    fin.append(curLine)
    fin[0] = fin[0].lstrip()
    fin.reverse()
    rt = '\n'.join(fin)
    return rt+'\n'

def slowPrint(text):
    for i in text.splitlines():
        printer.print(i+'\n')
        time.sleep(0.1)

def checkSensor():
    global rPast
    global rMin
    global rMax
    global noop
    # change this to whatever get-readings call we need
    r = adc.read(sensor_pin)
    readings.append(r)
    if len(readings)>WINDOW_SIZE:
        del readings[:-WINDOW_SIZE]

    avg = 0
    for i in readings:
        avg += (i/float(len(readings)))

    delta = r-rPast
    # delta = r-avg

    print('%0.3f\t%0.3f\t%0.3f\t%0.3f\t%0.3f'%(r, delta, avg, rMin, rMax))

    if r > rMax:
        rMax = r
        # does this merit an emission? Or should delta have to be > threshold?
    if r < rMin:
        rMin = r
        # does this merit an emission? Or should delta have to be > threshold?

    if abs(delta) > emission_threshold:
        if len(readings)==WINDOW_SIZE: # this test prevents emissions before WINDOW_SIZE samples have been taken
            # print('emitting remark')
            noop = 0
            emit_remark(r, delta, avg)
        else:
            pass
    else:
        noop += 1
        if noop > noop_threshold:
            noop = 0
            # print('emitting dream')
            emit_dream(r, delta, avg)
    rPast = r

def emit_dream(r, delta, avg):
    global fake
    if fake == 5:
        fake = 0
        norm = mapVals(r, rMin, rMax, 0.0, 0.999)
        sen = sg.generate(theObj, (1.0-norm), delta, True) # reverse the range so 1=no light, 0=full light

        # printer.flush()
        printer.feed(1)
        printer.print('            . . .             ')
        printer.feed(1)

        slowPrint(parse(sen))

        # printer.flush()
        printer.feed(1)
        printer.print('            . . .             ')
        printer.feed(2)
    else:
        fake += 1
        norm = mapVals(r, rMin, rMax, 0.0, 0.999)
        sen = sg.generate(theObj, (1.0-norm), delta, False) # reverse the range so 1=no light, 0=full light
        slowPrint(parse(sen))
        printer.feed(2)

def emit_remark(r, delta, avg):
    norm = mapVals(r, rMin, rMax, 1.0, 0.0)
    sen = sg.generate(theObj, (1.0-norm), delta, False) # reverse the range so 1=no light, 0=full light
    slowPrint(parse(sen))
    # slowPrint(str(norm))
    printer.feed(2)

def exit_handler():
    print('exiting')
    # adc.cleanup()
    # uart.cleanup() # not yet supported?

def mapVals(val, inMin, inMax, outMin, outMax):
        toRet = float(outMin + (float(outMax - outMin) * (float(val - inMin) / float(inMax - inMin))))
        toRet = clamp(toRet, outMin, outMax)
        print('turned %s into %s'%(val,toRet))
        return toRet

def clamp(val, tmin, tmax):
    if val > tmax:
        val = tmax
    if val < tmin:
        val = tmin
    return val

uart.setup("UART2")
adc.setup()
atexit.register(exit_handler)
printer = Adafruit_Thermal("/dev/ttyO2", 19200, timeout=5)
printer.begin()
printer.upsideDownOn()
printer.feed(3)
printer.print(parse('i am awake and I am APPLE (light)'))
printer.feed(1)
rPast = 0
rMax = 0 # all-time max sensor reading
rMin = 100000 # all-time min sensor reading
WINDOW_SIZE = 30 # size of moving-window avg
noop = 0 # number of intervals passed without a trigger
noop_threshold = 480
emission_threshold = 0.01 # changed this for vcnl4000, used to be 0.7

while True:
    checkSensor()
    time.sleep(0.5)<|endoftext|>