<commit_before>"""Test for letsencrypt_nginx.dvsni."""
import pkg_resources
import unittest
import shutil

import mock

from acme import challenges
from acme import messages2

from letsencrypt import achallenges
from letsencrypt import errors
from letsencrypt import le_util
from letsencrypt.tests import acme_util

from letsencrypt_nginx import obj
from letsencrypt_nginx.tests import util

class DvsniPerformTest(util.NginxTest):
    """Test the NginxDVSNI challenge."""

    def setUp(self):
        super(DvsniPerformTest, self).setUp()

        config = util.get_nginx_configurator(
            self.config_path, self.config_dir, self.work_dir,
            self.ssl_options)

        rsa256_file = pkg_resources.resource_filename(
            "acme.jose", "testdata/rsa256_key.pem")
        rsa256_pem = pkg_resources.resource_string(
            "acme.jose", "testdata/rsa256_key.pem")

        auth_key = le_util.Key(rsa256_file, rsa256_pem)

        from letsencrypt_nginx import dvsni
        self.sni = dvsni.NginxDvsni(config)

        self.achalls = [
            achallenges.DVSNI(
                challb=acme_util.chall_to_challb(
                    challenges.DVSNI(
                        r="foo",
                        nonce="bar"
                    ), "pending"),
                domain="www.example.com", key=auth_key),
            achallenges.DVSNI(
                challb=acme_util.chall_to_challb(
                    challenges.DVSNI(
                        r="\xba\xa9\xda?<m\xaewmx\xea\xad\xadv\xf4\x02\xc9y\x80"
                          "\xe2_X\t\xe7\xc7\xa4\t\xca\xf7&\x945",
                        nonce="Y\xed\x01L\xac\x95\xf7pW\xb1\xd7"
                              "\xa1\xb2\xc5\x96\xba"
                    ), "pending"),
                domain="blah", key=auth_key),
            achallenges.DVSNI(
                challb=acme_util.chall_to_challb(
                    challenges.DVSNI(
                        r="\x8c\x8a\xbf_-f\\cw\xee\xd6\xf8/\xa5\xe3\xfd\xeb9"
                          "\xf1\xf5\xb9\xefVM\xc9w\xa4u\x9c\xe1\x87\xb4",
                        nonce="7\xbc^\xb7]>\x00\xa1\x9bOcU\x84^Z\x18"
                    ), "pending"),
                domain="www.example.org", key=auth_key)
        ]

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    def test_add_chall(self):
        self.sni.add_chall(self.achalls[0], 0)
        self.assertEqual(1, len(self.sni.achalls))
        self.assertEqual([0], self.sni.indices)

    @mock.patch("letsencrypt_nginx.configurator"
                ".NginxConfigurator.choose_vhost")
    def test_perform(self, mock_choose):
        self.sni.add_chall(self.achalls[1])
        mock_choose.return_value = None
        result = self.sni.perform()
        self.assertTrue(result is None)

    def test_perform0(self):
        responses = self.sni.perform()
        self.assertEqual([], responses)

    @mock.patch("letsencrypt_nginx.configurator.NginxConfigurator.save")
    def test_perform1(self, mock_save):
        self.sni.add_chall(self.achalls[0])
        mock_setup_cert = mock.MagicMock(
            return_value=challenges.DVSNIResponse(s="nginxS1"))

        # pylint: disable=protected-access
        self.sni._setup_challenge_cert = mock_setup_cert

        responses = self.sni.perform()

        mock_setup_cert.assert_called_once_with(self.achalls[0])
        self.assertEqual([challenges.DVSNIResponse(s="nginxS1")], responses)
        self.assertEqual(mock_save.call_count, 2)

        # Make sure challenge config is included in main config
        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(['include', self.sni.challenge_conf] in http[1])

    def test_perform2(self):
        for achall in self.achalls:
            self.sni.add_chall(achall)

        mock_setup_cert = mock.MagicMock(side_effect=[
            challenges.DVSNIResponse(s="nginxS0"),
            challenges.DVSNIResponse(s="nginxS1"),
            challenges.DVSNIResponse(s="nginxS2")])
        # pylint: disable=protected-access
        self.sni._setup_challenge_cert = mock_setup_cert

        responses = self.sni.perform()

        self.assertEqual(mock_setup_cert.call_count, 3)

        for index, achall in enumerate(self.achalls):
            self.assertEqual(
                mock_setup_cert.call_args_list[index], mock.call(achall))

        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(['include', self.sni.challenge_conf] in http[1])
        self.assertTrue(['server_name', 'blah'] in http[1][-2][1])

        self.assertEqual(len(responses), 3)
        for i in xrange(3):
            self.assertEqual(responses[i].s, "nginxS%d" % i)

    def test_mod_config(self):
        self.sni.add_chall(self.achalls[0])
        self.sni.add_chall(self.achalls[2])

        v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False),
                   obj.Addr("127.0.0.1", "", False, False)]
        v_addr2 = [obj.Addr("myhost", "", False, True)]
        ll_addr = [v_addr1, v_addr2]
        self.sni._mod_config(ll_addr)  # pylint: disable=protected-access

        self.sni.configurator.save()

        self.sni.configurator.parser.load()

        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(['include', self.sni.challenge_conf] in http[1])

        vhosts = self.sni.configurator.parser.get_vhosts()
        vhs = [vh for vh in vhosts if vh.filep == self.sni.challenge_conf]

        for vhost in vhs:
            if vhost.addrs == set(v_addr1):
                self.assertEqual(
                    vhost.names, set([self.achalls[0].nonce_domain]))
            else:
                self.assertEqual(vhost.addrs, set(v_addr2))
                self.assertEqual(
                    vhost.names, set([self.achalls[2].nonce_domain]))

        self.assertEqual(len(vhs), 2)

    def test_mod_config_fail(self):
        root = self.sni.configurator.parser.loc["root"]
        self.sni.configurator.parser.parsed[root] = [['include', 'foo.conf']]
        # pylint: disable=protected-access
        self.assertRaises(errors.LetsEncryptMisconfigurationError,
                          self.sni._mod_config, [])

if __name__ == "__main__":
    unittest.main()  # pragma: no cover
<commit_msg>Remove unused import<commit_after>"""Test for letsencrypt_nginx.dvsni."""
import pkg_resources
import unittest
import shutil

import mock

from acme import challenges

from letsencrypt import achallenges
from letsencrypt import errors
from letsencrypt import le_util
from letsencrypt.tests import acme_util

from letsencrypt_nginx import obj
from letsencrypt_nginx.tests import util

class DvsniPerformTest(util.NginxTest):
    """Test the NginxDVSNI challenge."""

    def setUp(self):
        super(DvsniPerformTest, self).setUp()

        config = util.get_nginx_configurator(
            self.config_path, self.config_dir, self.work_dir,
            self.ssl_options)

        rsa256_file = pkg_resources.resource_filename(
            "acme.jose", "testdata/rsa256_key.pem")
        rsa256_pem = pkg_resources.resource_string(
            "acme.jose", "testdata/rsa256_key.pem")

        auth_key = le_util.Key(rsa256_file, rsa256_pem)

        from letsencrypt_nginx import dvsni
        self.sni = dvsni.NginxDvsni(config)

        self.achalls = [
            achallenges.DVSNI(
                challb=acme_util.chall_to_challb(
                    challenges.DVSNI(
                        r="foo",
                        nonce="bar"
                    ), "pending"),
                domain="www.example.com", key=auth_key),
            achallenges.DVSNI(
                challb=acme_util.chall_to_challb(
                    challenges.DVSNI(
                        r="\xba\xa9\xda?<m\xaewmx\xea\xad\xadv\xf4\x02\xc9y\x80"
                          "\xe2_X\t\xe7\xc7\xa4\t\xca\xf7&\x945",
                        nonce="Y\xed\x01L\xac\x95\xf7pW\xb1\xd7"
                              "\xa1\xb2\xc5\x96\xba"
                    ), "pending"),
                domain="blah", key=auth_key),
            achallenges.DVSNI(
                challb=acme_util.chall_to_challb(
                    challenges.DVSNI(
                        r="\x8c\x8a\xbf_-f\\cw\xee\xd6\xf8/\xa5\xe3\xfd\xeb9"
                          "\xf1\xf5\xb9\xefVM\xc9w\xa4u\x9c\xe1\x87\xb4",
                        nonce="7\xbc^\xb7]>\x00\xa1\x9bOcU\x84^Z\x18"
                    ), "pending"),
                domain="www.example.org", key=auth_key)
        ]

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        shutil.rmtree(self.config_dir)
        shutil.rmtree(self.work_dir)

    def test_add_chall(self):
        self.sni.add_chall(self.achalls[0], 0)
        self.assertEqual(1, len(self.sni.achalls))
        self.assertEqual([0], self.sni.indices)

    @mock.patch("letsencrypt_nginx.configurator"
                ".NginxConfigurator.choose_vhost")
    def test_perform(self, mock_choose):
        self.sni.add_chall(self.achalls[1])
        mock_choose.return_value = None
        result = self.sni.perform()
        self.assertTrue(result is None)

    def test_perform0(self):
        responses = self.sni.perform()
        self.assertEqual([], responses)

    @mock.patch("letsencrypt_nginx.configurator.NginxConfigurator.save")
    def test_perform1(self, mock_save):
        self.sni.add_chall(self.achalls[0])
        mock_setup_cert = mock.MagicMock(
            return_value=challenges.DVSNIResponse(s="nginxS1"))

        # pylint: disable=protected-access
        self.sni._setup_challenge_cert = mock_setup_cert

        responses = self.sni.perform()

        mock_setup_cert.assert_called_once_with(self.achalls[0])
        self.assertEqual([challenges.DVSNIResponse(s="nginxS1")], responses)
        self.assertEqual(mock_save.call_count, 2)

        # Make sure challenge config is included in main config
        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(['include', self.sni.challenge_conf] in http[1])

    def test_perform2(self):
        for achall in self.achalls:
            self.sni.add_chall(achall)

        mock_setup_cert = mock.MagicMock(side_effect=[
            challenges.DVSNIResponse(s="nginxS0"),
            challenges.DVSNIResponse(s="nginxS1"),
            challenges.DVSNIResponse(s="nginxS2")])
        # pylint: disable=protected-access
        self.sni._setup_challenge_cert = mock_setup_cert

        responses = self.sni.perform()

        self.assertEqual(mock_setup_cert.call_count, 3)

        for index, achall in enumerate(self.achalls):
            self.assertEqual(
                mock_setup_cert.call_args_list[index], mock.call(achall))

        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(['include', self.sni.challenge_conf] in http[1])
        self.assertTrue(['server_name', 'blah'] in http[1][-2][1])

        self.assertEqual(len(responses), 3)
        for i in xrange(3):
            self.assertEqual(responses[i].s, "nginxS%d" % i)

    def test_mod_config(self):
        self.sni.add_chall(self.achalls[0])
        self.sni.add_chall(self.achalls[2])

        v_addr1 = [obj.Addr("69.50.225.155", "9000", True, False),
                   obj.Addr("127.0.0.1", "", False, False)]
        v_addr2 = [obj.Addr("myhost", "", False, True)]
        ll_addr = [v_addr1, v_addr2]
        self.sni._mod_config(ll_addr)  # pylint: disable=protected-access

        self.sni.configurator.save()

        self.sni.configurator.parser.load()

        http = self.sni.configurator.parser.parsed[
            self.sni.configurator.parser.loc["root"]][-1]
        self.assertTrue(['include', self.sni.challenge_conf] in http[1])

        vhosts = self.sni.configurator.parser.get_vhosts()
        vhs = [vh for vh in vhosts if vh.filep == self.sni.challenge_conf]

        for vhost in vhs:
            if vhost.addrs == set(v_addr1):
                self.assertEqual(
                    vhost.names, set([self.achalls[0].nonce_domain]))
            else:
                self.assertEqual(vhost.addrs, set(v_addr2))
                self.assertEqual(
                    vhost.names, set([self.achalls[2].nonce_domain]))

        self.assertEqual(len(vhs), 2)

    def test_mod_config_fail(self):
        root = self.sni.configurator.parser.loc["root"]
        self.sni.configurator.parser.parsed[root] = [['include', 'foo.conf']]
        # pylint: disable=protected-access
        self.assertRaises(errors.LetsEncryptMisconfigurationError,
                          self.sni._mod_config, [])

if __name__ == "__main__":
    unittest.main()  # pragma: no cover
<|endoftext|>


<commit_before>import re
from typing import NamedTuple, Optional, List

import numpy as np

from lexos.helpers.definitions import WORD_AND_RIGHT_BOUNDARY_REGEX, \
    WORD_BOUNDARY_REGEX_STR
from lexos.models.base_model import BaseModel
from lexos.models.filemanager_model import FileManagerModel
from lexos.receivers.rolling_windows_receiver import RWAFrontEndOptions, \
    RollingWindowsReceiver, WindowUnitType, RWATokenType

rwa_regex_flags = re.DOTALL | re.MULTILINE | re.UNICODE

class RWATestOptions(NamedTuple):
    passage_string: str
    rolling_windows_options: RWAFrontEndOptions

class RollingWindowsModel(BaseModel):
    def __init__(self, test_option: Optional[RWATestOptions] = None):
        super().__init__()
        if test_option is not None:
            self._test_passage = test_option.passage_string
            self._test_front_end_options = test_option.rolling_windows_options
        else:
            self._test_passage = None
            self._test_front_end_options = None

    @property
    def _passage(self) -> str:
        if self._passage is not None:
            return self._passage
        else:
            file_id = RollingWindowsReceiver().get_file_id_from_front_end()
            file_id_content_map = FileManagerModel().load_file_manager() \
                .get_content_of_active_with_id()

            return file_id_content_map[file_id]

    @property
    def _options(self) -> RWAFrontEndOptions:
        return self._test_front_end_options \
            if self._test_front_end_options is not None \
            else RollingWindowsReceiver().options_from_front_end()

    @staticmethod
    def _get_rolling_window_from_list(input_list: List[str],
                                      window_size: int) -> np.ndarray:
        """

        :param input_list:
        :param window_size:
        """

        # number of items in the input list
        num_item = len(input_list)

        # get the total number of windows
        num_window = num_item - window_size

        # get the rolling list, should be a array of str
        return np.array(
            "".join(input_list[start: start + window_size])
            for start in range(num_window)
        )

    @staticmethod
    def _get_letters_windows(passage: str, windows_size: int) -> np.ndarray:
        """

        :param passage:
        :param windows_size:
        :return:
        """
        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=list(passage), window_size=windows_size
        )

    @staticmethod
    def _get_word_windows(passage: str, window_size: int) -> np.ndarray:
        """

        :param passage:
        :param window_size:
        :return:
        """

        words = re.findall(WORD_AND_RIGHT_BOUNDARY_REGEX, passage)

        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=words, window_size=window_size
        )

    @staticmethod
    def _get_line_windows(passage: str, window_size: int) -> np.ndarray:
        """

        :param passage:
        :param window_size:
        """

        # get all the lines
        lines = passage.splitlines(keepends=True)

        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=lines, window_size=window_size
        )

    @staticmethod
    def _find_regex_in_window(window: str, regex: str) -> int:
        return len(re.findall(pattern=regex, string=window,
                              flags=rwa_regex_flags))

    @staticmethod
    def _find_word_in_window(window: str, word: str) -> int:
        word_regex = re.compile(
            # enclose the word in word boundaries
            WORD_BOUNDARY_REGEX_STR + re.escape(word)
            + WORD_BOUNDARY_REGEX_STR,

            flags=rwa_regex_flags
        )

        return len(re.findall(pattern=word_regex, string=window))

    @staticmethod
    def _find_string_in_window(window: str, string: str) -> int:
        string_regex = re.compile(re.escape(string), flags=rwa_regex_flags)

        return len(re.findall(pattern=string_regex, string=window))

    def _get_window(self) -> np.ndarray:
        window_unit = self._options.window_options.window_unit
        window_size = self._options.window_options.window_size
        passage = self._passage

        if window_unit is WindowUnitType.line:
            return self._get_line_windows(passage=passage,
                                          window_size=window_size)

        elif window_unit is WindowUnitType.word:
            return self._get_word_windows(passage=passage,
                                          window_size=window_size)

        elif window_unit is WindowUnitType.letter:
            return self._get_letters_windows(passage=passage,
                                             windows_size=window_size)

        else:
            raise ValueError("unhandled window type: " + window_unit)

    def find_token_average_in_window(self, window: str) -> float:
        token_type = self._options.token_options.token_type
        token = self._options.token_options.token
        window_size = self._options.window_options.window_size

        if token_type is RWATokenType.string:
            token_count = self._find_string_in_window(window=window,
                                                      string=token)

        elif token_type is RWATokenType.word:
            token_count = self._find_word_in_window(window=window,
                                                    word=token)

        elif token_type is RWATokenType.regex:
            token_count = self._find_regex_in_window(window=window,
                                                     regex=token)

        else:
            raise ValueError("unhandled token type: " + token_type)

        return token_count / window_size

    def find_token_ratio_in_window(self, window: str) -> float:
        token_type = self._options.token_options.token_type
        primary_token = self._options.token_options.token
        secondary_token = self._options.token_options.secondary_token

        if token_type is RWATokenType.string:
            numerator = self._find_string_in_window(window=window,
                                                    string=primary_token)
            denominator = self._find_string_in_window(window=window,
                                                      string=secondary_token)

        elif token_type is RWATokenType.word:
            numerator = self._find_word_in_window(window=window,
                                                  word=primary_token)
            denominator = self._find_word_in_window(window=window,
                                                    word=secondary_token)

        elif token_type is RWATokenType.regex:
            numerator = self._find_regex_in_window(window=window,
                                                   regex=primary_token)
            denominator = self._find_regex_in_window(window=window,
                                                     regex=secondary_token)

        else:
            raise ValueError("unhandled token type: " + token_type)

        return numerator / denominator

<commit_msg>fix 2 of the find token in window function<commit_after>import re
from typing import NamedTuple, Optional, List

import numpy as np
import pandas as pd

from lexos.helpers.definitions import WORD_AND_RIGHT_BOUNDARY_REGEX, \
    WORD_BOUNDARY_REGEX_STR
from lexos.models.base_model import BaseModel
from lexos.models.filemanager_model import FileManagerModel
from lexos.receivers.rolling_windows_receiver import RWAFrontEndOptions, \
    RollingWindowsReceiver, WindowUnitType, RWATokenType

rwa_regex_flags = re.DOTALL | re.MULTILINE | re.UNICODE

class RWATestOptions(NamedTuple):
    passage_string: str
    rolling_windows_options: RWAFrontEndOptions

class RollingWindowsModel(BaseModel):
    def __init__(self, test_option: Optional[RWATestOptions] = None):
        super().__init__()
        if test_option is not None:
            self._test_passage = test_option.passage_string
            self._test_front_end_options = test_option.rolling_windows_options
        else:
            self._test_passage = None
            self._test_front_end_options = None

    @property
    def _passage(self) -> str:
        if self._passage is not None:
            return self._passage
        else:
            file_id = RollingWindowsReceiver().get_file_id_from_front_end()
            file_id_content_map = FileManagerModel().load_file_manager() \
                .get_content_of_active_with_id()

            return file_id_content_map[file_id]

    @property
    def _options(self) -> RWAFrontEndOptions:
        return self._test_front_end_options \
            if self._test_front_end_options is not None \
            else RollingWindowsReceiver().options_from_front_end()

    @staticmethod
    def _get_rolling_window_from_list(input_list: List[str],
                                      window_size: int) -> np.ndarray:
        """

        :param input_list:
        :param window_size:
        """

        # number of items in the input list
        num_item = len(input_list)

        # get the total number of windows
        num_window = num_item - window_size

        # get the rolling list, should be a array of str
        return np.array(
            "".join(input_list[start: start + window_size])
            for start in range(num_window)
        )

    @staticmethod
    def _get_letters_windows(passage: str, windows_size: int) -> np.ndarray:
        """

        :param passage:
        :param windows_size:
        :return:
        """
        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=list(passage), window_size=windows_size
        )

    @staticmethod
    def _get_word_windows(passage: str, window_size: int) -> np.ndarray:
        """

        :param passage:
        :param window_size:
        :return:
        """

        words = re.findall(WORD_AND_RIGHT_BOUNDARY_REGEX, passage)

        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=words, window_size=window_size
        )

    @staticmethod
    def _get_line_windows(passage: str, window_size: int) -> np.ndarray:
        """

        :param passage:
        :param window_size:
        """

        # get all the lines
        lines = passage.splitlines(keepends=True)

        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=lines, window_size=window_size
        )

    @staticmethod
    def _find_regex_in_window(window: str, regex: str) -> int:
        return len(re.findall(pattern=regex, string=window,
                              flags=rwa_regex_flags))

    @staticmethod
    def _find_word_in_window(window: str, word: str) -> int:
        word_regex = re.compile(
            # enclose the word in word boundaries
            WORD_BOUNDARY_REGEX_STR + re.escape(word)
            + WORD_BOUNDARY_REGEX_STR,

            flags=rwa_regex_flags
        )

        return len(re.findall(pattern=word_regex, string=window))

    @staticmethod
    def _find_string_in_window(window: str, string: str) -> int:
        string_regex = re.compile(re.escape(string), flags=rwa_regex_flags)

        return len(re.findall(pattern=string_regex, string=window))

    def _get_window(self) -> np.ndarray:
        window_unit = self._options.window_options.window_unit
        window_size = self._options.window_options.window_size
        passage = self._passage

        if window_unit is WindowUnitType.line:
            return self._get_line_windows(passage=passage,
                                          window_size=window_size)

        elif window_unit is WindowUnitType.word:
            return self._get_word_windows(passage=passage,
                                          window_size=window_size)

        elif window_unit is WindowUnitType.letter:
            return self._get_letters_windows(passage=passage,
                                             windows_size=window_size)

        else:
            raise ValueError("unhandled window type: " + window_unit)

    def find_token_average_in_window(self, window: str) -> pd.Series:

        assert self._options.average_token_options is not None

        token_type = self._options.average_token_options.token_type
        tokens = self._options.average_token_options.tokens
        window_size = self._options.window_options.window_size

        if token_type is RWATokenType.string:
            token_counts = [
                self._find_string_in_window(window=window, string=token)
                for token in tokens
            ]

        elif token_type is RWATokenType.word:
            token_counts = [
                self._find_word_in_window(window=window, word=token)
                for token in tokens
            ]

        elif token_type is RWATokenType.regex:
            token_counts = [
                self._find_regex_in_window(window=window, regex=token)
                for token in tokens
            ]

        else:
            raise ValueError("unhandled token type: " + token_type)

        return pd.Series(
            [token_count / window_size for token_count in token_counts],
            index=tokens
        )

    def find_token_ratio_in_window(self, window: str) -> float:

        assert self._options.average_token_options is not None

        token_type = self._options.ratio_token_options.token_type
        numerator_token = self._options.ratio_token_options.numerator_token
        denominator_token = self._options.ratio_token_options.denominator_token

        if token_type is RWATokenType.string:
            numerator = self._find_string_in_window(window=window,
                                                    string=numerator_token)
            denominator = self._find_string_in_window(window=window,
                                                      string=denominator_token)

        elif token_type is RWATokenType.word:
            numerator = self._find_word_in_window(window=window,
                                                  word=numerator_token)
            denominator = self._find_word_in_window(window=window,
                                                    word=denominator_token)

        elif token_type is RWATokenType.regex:
            numerator = self._find_regex_in_window(window=window,
                                                   regex=numerator_token)
            denominator = self._find_regex_in_window(window=window,
                                                     regex=denominator_token)

        else:
            raise ValueError("unhandled token type: " + token_type)

        return numerator / denominator
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

#
# Xibo - Digitial Signage - http://www.xibo.org.uk
# Copyright (C) 2009 Alex Harrington
#
# This file is part of Xibo.
#
# Xibo is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# Xibo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Xibo.  If not, see <http://www.gnu.org/licenses/>.
#

from VideoMedia import VideoMedia
from threading import Thread
import os

class LocalvideoMedia(VideoMedia):
    def add(self):
        video = self.options['uri']
        tmpXML = str('<video href="%s" id="%s" opacity="0" />' % (video,self.mediaNodeName))
        self.p.enqueue('add',(tmpXML,self.regionNodeName))

    def requiredFiles(self):
        # We have to assume that the video will be where it's supposed to be!
        return []
<commit_msg>[pyclient] Fixed decoding local video URL when the server sends it to us urlencoded<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

#
# Xibo - Digitial Signage - http://www.xibo.org.uk
# Copyright (C) 2009-13 Alex Harrington
#
# This file is part of Xibo.
#
# Xibo is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# Xibo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Xibo.  If not, see <http://www.gnu.org/licenses/>.
#

from VideoMedia import VideoMedia
from threading import Thread
import os
import urllib

class LocalvideoMedia(VideoMedia):
    def add(self):
        video = urllib.unquote_plus(self.options['uri'])
        tmpXML = str('<video href="%s" id="%s" opacity="0" />' % (video,self.mediaNodeName))
        self.p.enqueue('add',(tmpXML,self.regionNodeName))

    def requiredFiles(self):
        # We have to assume that the video will be where it's supposed to be!
        return []
<|endoftext|>


<commit_before>import re
from typing import NamedTuple, Optional, List, Iterator, Callable, Union

import numpy as np
import pandas as pd
import plotly.graph_objs as go
from plotly.offline import plot

from lexos.helpers.definitions import get_words_with_right_boundary, \
    get_single_word_count_in_text
from lexos.models.base_model import BaseModel
from lexos.models.filemanager_model import FileManagerModel
from lexos.models.matrix_model import FileIDContentMap
from lexos.receivers.rolling_windows_receiver import RWAFrontEndOptions, \
    RollingWindowsReceiver, WindowUnitType, RWATokenType

rwa_regex_flags = re.DOTALL | re.MULTILINE | re.UNICODE

class RWATestOptions(NamedTuple):
    file_id_content_map: FileIDContentMap
    rolling_windows_options: RWAFrontEndOptions

class RollingWindowsModel(BaseModel):
    def __init__(self, test_option: Optional[RWATestOptions] = None):
        """The class for rolling window calculation.

        :param test_option: the options to send in for testing
        """
        super().__init__()
        if test_option is not None:
            self._test_file_id_content_map = test_option.file_id_content_map
            self._test_front_end_options = test_option.rolling_windows_options
        else:
            self._test_file_id_content_map = None
            self._test_front_end_options = None

    @property
    def _passage(self) -> str:
        """The passage to run rolling windows on

        :return: the content of the passage as a string
        """
        # if test option is specified
        if self._test_file_id_content_map is not None and \
                self._test_front_end_options is not None:
            file_id = self._test_front_end_options.passage_file_id
            file_id_content_map = self._test_file_id_content_map

        # if test option is not specified, get option from front end
        else:
            file_id = RollingWindowsReceiver().options_from_front_end()\
                .passage_file_id
            file_id_content_map = FileManagerModel().load_file_manager() \
                .get_content_of_active_with_id()

        return file_id_content_map[file_id]

    @property
    def _options(self) -> RWAFrontEndOptions:
        """the front end option packed into one named tuple

        :return: a RWAFrontEndOption packs all the frontend option
        """
        return self._test_front_end_options \
            if self._test_front_end_options is not None \
            else RollingWindowsReceiver().options_from_front_end()

    @staticmethod
    def _get_rolling_window_from_list(input_list: List[str],
                                      window_size: int) -> np.ndarray:
        """A helper function to get the rolling window from the list of terms

        :param input_list: a list of terms (word, char or line),
            depends on the window type (word and line are with endings)
        :param window_size: the size of the window (number of terms in window)
        :return: an array of strings, each element is a window
        """

        # number of items in the input list
        num_item = len(input_list)

        # get the total number of windows
        num_window = num_item - window_size + 1

        # get the rolling list, should be a array of str
        return np.array([
            "".join(input_list[start: start + window_size])
            for start in range(num_window)
        ])

    @staticmethod
    def _get_letters_windows(passage: str, windows_size: int) -> np.ndarray:
        """Get the windows of letters with specific window size

        :param passage: the whole text to generate the windows
            (the text to run rolling window analysis on)
        :param windows_size: number of terms (letters) in a single window
        :return: an array of windows
        """
        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=list(passage), window_size=windows_size
        )

    @staticmethod
    def _get_word_windows(passage: str, window_size: int) -> np.ndarray:
        """Get the window of words with specific window size

        :param passage: the whole text to generate the windows
            (the text to run rolling window analysis on)
        :param window_size: number of terms (words) in a single window
        :return: an array of windows
        """

        words = get_words_with_right_boundary(passage)

        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=words, window_size=window_size
        )

    @staticmethod
    def _get_line_windows(passage: str, window_size: int) -> np.ndarray:
        """Get the window of lines with specific size

        :param passage: the whole text ot generate the windows
            (the text to run rolling window analysis on)
        :param window_size: the number of terms (lines) in a window
        :return: an array of windows
        """

        # get all the lines
        lines = passage.splitlines(keepends=True)

        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=lines, window_size=window_size
        )

    @staticmethod
    def _find_regex_in_window(window: str, regex: str) -> int:
        """find the number of times the regex appear in window

        current method only finds non-overlapping regex.
        :param window: find regex in this window
        :param regex: the regex to find
        :return: the number of times the regex appear in the window
        """
        return len(re.findall(pattern=regex, string=window,
                              flags=rwa_regex_flags))

    @staticmethod
    def _find_word_in_window(window: str, word: str) -> int:
        """find the number of times a particular word appear in the window

        :param window: find the word in this window
        :param word: the word to find
        :return: the number of times the word appear int the window
        """
        return get_single_word_count_in_text(text=window, word=word)

    @staticmethod
    def _find_string_in_window(window: str, string: str) -> int:
        """find the number of times the string appear in the window

        :param window: find the string in this window
        :param string: the string to find
        :return: the number of times the string appear int the window
        """
        string_regex = re.compile(re.escape(string), flags=rwa_regex_flags)

        return len(re.findall(pattern=string_regex, string=window))

    def _get_windows(self) -> np.ndarray:
        """get the array of window with the option in classes

        :return: an array of windows to run analysis on
        """
        window_unit = self._options.window_options.window_unit
        window_size = self._options.window_options.window_size
        passage = self._passage

        if window_unit is WindowUnitType.line:
            return self._get_line_windows(passage=passage,
                                          window_size=window_size)

        elif window_unit is WindowUnitType.word:
            return self._get_word_windows(passage=passage,
                                          window_size=window_size)

        elif window_unit is WindowUnitType.letter:
            return self._get_letters_windows(passage=passage,
                                             windows_size=window_size)

        else:
            raise ValueError(f"unhandled window type: {window_unit}")

    def _find_tokens_average_in_windows(self, windows: Iterator[str]) \
            -> pd.DataFrame:
        """find the token average in the given windows

        a token average is calculated by the number of times the token
        (or term) appear in the window divided by the window size
        :param windows: an array of windows to calculate
        :return: a panda data frame where:
            - the index header is the tokens
            - the column header corresponds to the windows but there is
                no column header, because it is impossible to set the header
                as windows
        """
        assert self._options.average_token_options is not None

        token_type = self._options.average_token_options.token_type
        tokens = self._options.average_token_options.tokens
        window_size = self._options.window_options.window_size

        def _average_matrix_helper(
                window_term_count_func: Callable[[str, str], int]) \
                -> pd.DataFrame:
            """The helper to get the average matrix

            :param window_term_count_func:
                the function to get count of term in the window
                the window is the first argument
                the term is the second argument,
                returns an int, that is the count of the term in the window
            :return: a panda data frame where
                - the index header is the tokens
                - the column header corresponds to the windows but
                    but there is no column header, because it is impossible to
                    set the header as windows
            """
            # we cannot use keyword parameter in window_term_count_func
            # because:
            #  - the type hinting does not support keyword parameter
            #       (on Python 3.6.1)
            #  - the function that sent in has different keywords
            list_matrix = [[window_term_count_func(window, token) / window_size
                            for token in tokens] for window in windows]

            return pd.DataFrame(list_matrix, columns=tokens).transpose()

        if token_type is RWATokenType.string:
            return _average_matrix_helper(self._find_string_in_window)

        elif token_type is RWATokenType.word:
            return _average_matrix_helper(self._find_word_in_window)

        elif token_type is RWATokenType.regex:
            return _average_matrix_helper(self._find_regex_in_window)

        else:
            raise ValueError(f"unhandled token type: {token_type}")

    def _find_token_ratio_in_windows(self, windows: Iterator[str]) \
            -> pd.Series:
        """Find the token ratios in all the windows

        get the ratio of the count of the numerator token and denominator token
        if the count of denominator token for that window is 0,
        that window's data will be np.nan
        :param windows: all the windows to get the ratio
        :return: a series of ratio, the index correspond to the windows
        """
        assert self._options.ratio_token_options is not None

        token_type = self._options.ratio_token_options.token_type
        numerator_token = self._options.ratio_token_options.numerator_token
        denominator_token = self._options.ratio_token_options.denominator_token

        def _get_ratio_helper(
                window: str,
                window_term_count_func: Callable[[str, str], int]) -> float:
            """the helper method to find a ratio for a single window

            :param window: the window to find ratio in
            :param window_term_count_func:
                the function to get count of term in the window
                the window is the first argument
                the term is the second argument,
                returns an int, that is the count of the term in the window
            :return: a float represent the ratio of the count of
                the nominator token and denominator token.
            """
            # we cannot use keyword parameter on window_term_count_func
            # because:
            #  - the type hinting does not support keyword parameter
            #       (on Python 3.6.1)
            #  - the function that sent in has different keywords
            numerator = window_term_count_func(window, numerator_token)
            denominator = window_term_count_func(window, denominator_token)

            # handle division by 0
            if denominator == 0:
                return np.nan
            else:
                return numerator / denominator

        if token_type is RWATokenType.string:
            return pd.Series(
                # the list to pack into the series
                [
                    _get_ratio_helper(
                        window=window,
                        window_term_count_func=self._find_string_in_window)
                    for window in windows
                ],
                # the name of the series
                name=f"{numerator_token} / {denominator_token}"
            )

        elif token_type is RWATokenType.word:
            return pd.Series(
                # the list to pack into the series
                [
                    _get_ratio_helper(
                        window=window,
                        window_term_count_func=self._find_word_in_window)
                    for window in windows
                ],
                # the name of the series
                name=f"{numerator_token} / {denominator_token}"
            )

        elif token_type is RWATokenType.regex:
            return pd.Series(
                # the list to pack into the series
                [
                    _get_ratio_helper(
                        window=window,
                        window_term_count_func=self._find_regex_in_window)
                    for window in windows
                ],
                # the name of the series
                name=f"{numerator_token} / {denominator_token}"
            )

        else:
            raise ValueError(f"unhandled token type: {token_type}")

    def _find_mile_stone_windows_indexes_in_all_windows(
            self, windows: Iterator[str]) -> List[int]:
        """Get a indexes of the mile stone windows

        A "mile stone window" is a window where the window that starts with
        the milestone string.
        :param windows: a iterator of windows
        :return: a list of indexes of the mile stone windows
        """

        # get an empty graph if the milestone is empty
        if self._options.milestone is None:
            return []

        # if the milestone string exists
        else:
            # get the mile stone str
            milestone_str = self._options.milestone

            # get the index fo the mile stone window
            return [index for index, window in enumerate(windows)
                    if window.startswith(milestone_str)]

    def _get_token_ratio_graph(self) -> go.Scattergl:
        """Get the plotly graph for the token ratio without milestone.

        :return: plotly graph object
        """

        windows = self._get_windows()

        token_ratio_series = self._find_token_ratio_in_windows(windows)

        # TODO: support black and white color scheme
        # TODO: support show dots, (just change the mode)
        # construct the graph object
        return go.Scattergl(
            # the x coordinates are the index of the window, starting from 0
            x=np.arange(len(token_ratio_series)),
            # the y coordinates is the token ratios
            y=token_ratio_series,
            mode="lines",
            name=token_ratio_series.name
        )

    def _get_token_average_graph(self) -> List[go.Scattergl]:
        """Get the plotly graph for token average without milestone.

        :return: a list of plotly graph object
        """
        windows = self._get_windows()

        token_average_data_frame = self._find_tokens_average_in_windows(
            windows=windows)

        # TODO: support black and white color scheme
        # TODO: support show dots, (just change the mode)
        return [
            go.Scattergl(
                x=np.arange(len(row)),
                y=row,
                mode="lines",
                name=token,
                marker=dict(
                    line=dict(
                        colorscale=[[0, 'rgb(0,0,0)', [1, 'rgb(128,128,128)']]]
                    )
                )
            ) for token, row in token_average_data_frame.iterrows()
        ]

    def get_rwa_graph(self) -> Union[List[go.Scattergl], go.Scattergl]:
        """Get the rolling window graph

        :return: a plotly scatter object or a list of plotly scatter objects.
        """
        count_average = self._options.average_token_options is not None
        count_ratio = self._options.ratio_token_options is not None

        # precondition
        # ^ is the exclusive or operator,
        # means we can either use average count or ratio count
        assert count_average ^ count_ratio

        if count_average:
            result_plot = self._get_token_average_graph()
        elif count_ratio:
            result_plot = self._get_token_ratio_graph()
        else:
            raise ValueError("unhandled count type")

        return plot(result_plot, include_plotlyjs=False, output_type='div')

    def _get_rwa_graph(self) -> str:

        return plot(
            self._get_rwa_plotly(), include_plotlyjs=False, output_type='div',
            validate=False
        )
<commit_msg>working with color scale rw model<commit_after>import re
from typing import NamedTuple, Optional, List, Iterator, Callable, Union

import numpy as np
import pandas as pd
import plotly.graph_objs as go
from plotly.offline import plot

from lexos.helpers.definitions import get_words_with_right_boundary, \
    get_single_word_count_in_text
from lexos.models.base_model import BaseModel
from lexos.models.filemanager_model import FileManagerModel
from lexos.models.matrix_model import FileIDContentMap
from lexos.receivers.rolling_windows_receiver import RWAFrontEndOptions, \
    RollingWindowsReceiver, WindowUnitType, RWATokenType

rwa_regex_flags = re.DOTALL | re.MULTILINE | re.UNICODE

class RWATestOptions(NamedTuple):
    file_id_content_map: FileIDContentMap
    rolling_windows_options: RWAFrontEndOptions

class RollingWindowsModel(BaseModel):
    def __init__(self, test_option: Optional[RWATestOptions] = None):
        """The class for rolling window calculation.

        :param test_option: the options to send in for testing
        """
        super().__init__()
        if test_option is not None:
            self._test_file_id_content_map = test_option.file_id_content_map
            self._test_front_end_options = test_option.rolling_windows_options
        else:
            self._test_file_id_content_map = None
            self._test_front_end_options = None

    @property
    def _passage(self) -> str:
        """The passage to run rolling windows on

        :return: the content of the passage as a string
        """
        # if test option is specified
        if self._test_file_id_content_map is not None and \
                self._test_front_end_options is not None:
            file_id = self._test_front_end_options.passage_file_id
            file_id_content_map = self._test_file_id_content_map

        # if test option is not specified, get option from front end
        else:
            file_id = RollingWindowsReceiver().options_from_front_end()\
                .passage_file_id
            file_id_content_map = FileManagerModel().load_file_manager() \
                .get_content_of_active_with_id()

        return file_id_content_map[file_id]

    @property
    def _options(self) -> RWAFrontEndOptions:
        """the front end option packed into one named tuple

        :return: a RWAFrontEndOption packs all the frontend option
        """
        return self._test_front_end_options \
            if self._test_front_end_options is not None \
            else RollingWindowsReceiver().options_from_front_end()

    @staticmethod
    def _get_rolling_window_from_list(input_list: List[str],
                                      window_size: int) -> np.ndarray:
        """A helper function to get the rolling window from the list of terms

        :param input_list: a list of terms (word, char or line),
            depends on the window type (word and line are with endings)
        :param window_size: the size of the window (number of terms in window)
        :return: an array of strings, each element is a window
        """

        # number of items in the input list
        num_item = len(input_list)

        # get the total number of windows
        num_window = num_item - window_size + 1

        # get the rolling list, should be a array of str
        return np.array([
            "".join(input_list[start: start + window_size])
            for start in range(num_window)
        ])

    @staticmethod
    def _get_letters_windows(passage: str, windows_size: int) -> np.ndarray:
        """Get the windows of letters with specific window size

        :param passage: the whole text to generate the windows
            (the text to run rolling window analysis on)
        :param windows_size: number of terms (letters) in a single window
        :return: an array of windows
        """
        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=list(passage), window_size=windows_size
        )

    @staticmethod
    def _get_word_windows(passage: str, window_size: int) -> np.ndarray:
        """Get the window of words with specific window size

        :param passage: the whole text to generate the windows
            (the text to run rolling window analysis on)
        :param window_size: number of terms (words) in a single window
        :return: an array of windows
        """

        words = get_words_with_right_boundary(passage)

        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=words, window_size=window_size
        )

    @staticmethod
    def _get_line_windows(passage: str, window_size: int) -> np.ndarray:
        """Get the window of lines with specific size

        :param passage: the whole text ot generate the windows
            (the text to run rolling window analysis on)
        :param window_size: the number of terms (lines) in a window
        :return: an array of windows
        """

        # get all the lines
        lines = passage.splitlines(keepends=True)

        return RollingWindowsModel._get_rolling_window_from_list(
            input_list=lines, window_size=window_size
        )

    @staticmethod
    def _find_regex_in_window(window: str, regex: str) -> int:
        """find the number of times the regex appear in window

        current method only finds non-overlapping regex.
        :param window: find regex in this window
        :param regex: the regex to find
        :return: the number of times the regex appear in the window
        """
        return len(re.findall(pattern=regex, string=window,
                              flags=rwa_regex_flags))

    @staticmethod
    def _find_word_in_window(window: str, word: str) -> int:
        """find the number of times a particular word appear in the window

        :param window: find the word in this window
        :param word: the word to find
        :return: the number of times the word appear int the window
        """
        return get_single_word_count_in_text(text=window, word=word)

    @staticmethod
    def _find_string_in_window(window: str, string: str) -> int:
        """find the number of times the string appear in the window

        :param window: find the string in this window
        :param string: the string to find
        :return: the number of times the string appear int the window
        """
        string_regex = re.compile(re.escape(string), flags=rwa_regex_flags)

        return len(re.findall(pattern=string_regex, string=window))

    def _get_windows(self) -> np.ndarray:
        """get the array of window with the option in classes

        :return: an array of windows to run analysis on
        """
        window_unit = self._options.window_options.window_unit
        window_size = self._options.window_options.window_size
        passage = self._passage

        if window_unit is WindowUnitType.line:
            return self._get_line_windows(passage=passage,
                                          window_size=window_size)

        elif window_unit is WindowUnitType.word:
            return self._get_word_windows(passage=passage,
                                          window_size=window_size)

        elif window_unit is WindowUnitType.letter:
            return self._get_letters_windows(passage=passage,
                                             windows_size=window_size)

        else:
            raise ValueError(f"unhandled window type: {window_unit}")

    def _find_tokens_average_in_windows(self, windows: Iterator[str]) \
            -> pd.DataFrame:
        """find the token average in the given windows

        a token average is calculated by the number of times the token
        (or term) appear in the window divided by the window size
        :param windows: an array of windows to calculate
        :return: a panda data frame where:
            - the index header is the tokens
            - the column header corresponds to the windows but there is
                no column header, because it is impossible to set the header
                as windows
        """
        assert self._options.average_token_options is not None

        token_type = self._options.average_token_options.token_type
        tokens = self._options.average_token_options.tokens
        window_size = self._options.window_options.window_size

        def _average_matrix_helper(
                window_term_count_func: Callable[[str, str], int]) \
                -> pd.DataFrame:
            """The helper to get the average matrix

            :param window_term_count_func:
                the function to get count of term in the window
                the window is the first argument
                the term is the second argument,
                returns an int, that is the count of the term in the window
            :return: a panda data frame where
                - the index header is the tokens
                - the column header corresponds to the windows but
                    but there is no column header, because it is impossible to
                    set the header as windows
            """
            # we cannot use keyword parameter in window_term_count_func
            # because:
            #  - the type hinting does not support keyword parameter
            #       (on Python 3.6.1)
            #  - the function that sent in has different keywords
            list_matrix = [[window_term_count_func(window, token) / window_size
                            for token in tokens] for window in windows]

            return pd.DataFrame(list_matrix, columns=tokens).transpose()

        if token_type is RWATokenType.string:
            return _average_matrix_helper(self._find_string_in_window)

        elif token_type is RWATokenType.word:
            return _average_matrix_helper(self._find_word_in_window)

        elif token_type is RWATokenType.regex:
            return _average_matrix_helper(self._find_regex_in_window)

        else:
            raise ValueError(f"unhandled token type: {token_type}")

    def _find_token_ratio_in_windows(self, windows: Iterator[str]) \
            -> pd.Series:
        """Find the token ratios in all the windows

        get the ratio of the count of the numerator token and denominator token
        if the count of denominator token for that window is 0,
        that window's data will be np.nan
        :param windows: all the windows to get the ratio
        :return: a series of ratio, the index correspond to the windows
        """
        assert self._options.ratio_token_options is not None

        token_type = self._options.ratio_token_options.token_type
        numerator_token = self._options.ratio_token_options.numerator_token
        denominator_token = self._options.ratio_token_options.denominator_token

        def _get_ratio_helper(
                window: str,
                window_term_count_func: Callable[[str, str], int]) -> float:
            """the helper method to find a ratio for a single window

            :param window: the window to find ratio in
            :param window_term_count_func:
                the function to get count of term in the window
                the window is the first argument
                the term is the second argument,
                returns an int, that is the count of the term in the window
            :return: a float represent the ratio of the count of
                the nominator token and denominator token.
            """
            # we cannot use keyword parameter on window_term_count_func
            # because:
            #  - the type hinting does not support keyword parameter
            #       (on Python 3.6.1)
            #  - the function that sent in has different keywords
            numerator = window_term_count_func(window, numerator_token)
            denominator = window_term_count_func(window, denominator_token)

            # handle division by 0
            if denominator == 0:
                return np.nan
            else:
                return numerator / denominator

        if token_type is RWATokenType.string:
            return pd.Series(
                # the list to pack into the series
                [
                    _get_ratio_helper(
                        window=window,
                        window_term_count_func=self._find_string_in_window)
                    for window in windows
                ],
                # the name of the series
                name=f"{numerator_token} / {denominator_token}"
            )

        elif token_type is RWATokenType.word:
            return pd.Series(
                # the list to pack into the series
                [
                    _get_ratio_helper(
                        window=window,
                        window_term_count_func=self._find_word_in_window)
                    for window in windows
                ],
                # the name of the series
                name=f"{numerator_token} / {denominator_token}"
            )

        elif token_type is RWATokenType.regex:
            return pd.Series(
                # the list to pack into the series
                [
                    _get_ratio_helper(
                        window=window,
                        window_term_count_func=self._find_regex_in_window)
                    for window in windows
                ],
                # the name of the series
                name=f"{numerator_token} / {denominator_token}"
            )

        else:
            raise ValueError(f"unhandled token type: {token_type}")

    def _find_mile_stone_windows_indexes_in_all_windows(
            self, windows: Iterator[str]) -> List[int]:
        """Get a indexes of the mile stone windows

        A "mile stone window" is a window where the window that starts with
        the milestone string.
        :param windows: a iterator of windows
        :return: a list of indexes of the mile stone windows
        """

        # get an empty graph if the milestone is empty
        if self._options.milestone is None:
            return []

        # if the milestone string exists
        else:
            # get the mile stone str
            milestone_str = self._options.milestone

            # get the index fo the mile stone window
            return [index for index, window in enumerate(windows)
                    if window.startswith(milestone_str)]

    def _get_token_ratio_graph(self) -> go.Scattergl:
        """Get the plotly graph for the token ratio without milestone.

        :return: plotly graph object
        """

        windows = self._get_windows()

        token_ratio_series = self._find_token_ratio_in_windows(windows)

        # TODO: support black and white color scheme
        # TODO: support show dots, (just change the mode)
        # construct the graph object
        return go.Scattergl(
            # the x coordinates are the index of the window, starting from 0
            x=np.arange(len(token_ratio_series)),
            # the y coordinates is the token ratios
            y=token_ratio_series,
            mode="lines",
            name=token_ratio_series.name
        )

    def _get_token_average_graph(self) -> List[go.Scattergl]:
        """Get the plotly graph for token average without milestone.

        :return: a list of plotly graph object
        """
        windows = self._get_windows()

        token_average_data_frame = self._find_tokens_average_in_windows(
            windows=windows)

        # TODO: support black and white color scheme
        # TODO: support show dots, (just change the mode)
        return [
            go.Scattergl(
                x=np.arange(len(row)),
                y=row,
                name=token,
                marker=dict(
                    line=dict(
                        cmax=39,
                        cmin=0,
                        color=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39],
                        colorscale="Greys",
                        width=4)
                ),
                mode="lines",
            ) for token, row in token_average_data_frame.iterrows()
        ]

    def get_rwa_graph(self) -> Union[List[go.Scattergl], go.Scattergl]:
        """Get the rolling window graph

        :return: a plotly scatter object or a list of plotly scatter objects.
        """
        count_average = self._options.average_token_options is not None
        count_ratio = self._options.ratio_token_options is not None

        # precondition
        # ^ is the exclusive or operator,
        # means we can either use average count or ratio count
        assert count_average ^ count_ratio

        if count_average:
            result_plot = self._get_token_average_graph()
        elif count_ratio:
            result_plot = self._get_token_ratio_graph()
        else:
            raise ValueError("unhandled count type")

        return plot(result_plot, include_plotlyjs=False, output_type='div')

    def _get_rwa_graph(self) -> str:

        return plot(
            self._get_rwa_plotly(), include_plotlyjs=False, output_type='div',
            validate=False
        )
<|endoftext|>


<commit_before>import sys
import time
import datetime
import argparse
import struct

import serial
import pygame
from pygame.locals import *

sys.path.append('../lib')
import hdlc

FIX_DIV = 65536.0

parser = argparse.ArgumentParser(description="RustTelemetry", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--port", type=str, default="/dev/tty.usbserial-A8008iwL", help="usb serial port device eg. /dev/ttyUSB0")
args = parser.parse_args()

s = serial.Serial(args.port, 57600, timeout=0.1);
s.flushInput()
s.flushOutput()

parser = hdlc.HdlcChecksummed()

# header bytes
BC_TELEMETRY = 0x01
CB_MOTOR_COMMAND = 0x02

AUTOMATIC_DEFAULT = 0
STEERING_PWM_DEFAULT = 90
DRIVING_PWM_DEFAULT = 92

def send_packet(data):
	data = hdlc.add_checksum(data)
	data = hdlc.escape_delimit(data)
	s.write(data)

def run():
	pygame.init()
	pygame.display.set_caption("RustTelemetry")

	screen = pygame.display.set_mode((600,480))
	# by default the key repeat is disabled, enable it
	pygame.key.set_repeat(50, 50)

	running = True
	automatic = AUTOMATIC_DEFAULT
	steering_pwm = STEERING_PWM_DEFAULT # center
	drive_pwm = DRIVING_PWM_DEFAULT # stop

	while running:
		for event in pygame.event.get():
			if event.type == pygame.QUIT:
				running = False
			elif event.type == KEYUP:
				if event.key == K_SPACE:
					automatic = 0 if automatic == 1 else 1
					steering_pwm = STEERING_PWM_DEFAULT # center
					drive_pwm = DRIVING_PWM_DEFAULT # stop
			elif event.type == KEYDOWN:
				if event.key == K_RIGHT:
					steering_pwm += 1
					if steering_pwm > 180:
						steering_pwm = 180
					print 'steering pwm %u' % steering_pwm
				elif event.key == K_LEFT:
					steering_pwm -= 1
					if steering_pwm < 0:
						steering_pwm = 0
					print 'steering pwm %u' % steering_pwm
				elif event.key == K_UP:
					drive_pwm += 1
					if drive_pwm > 180:
						drive_pwm = 180
					print 'drive pwm %u' % drive_pwm
				elif event.key == K_DOWN:
					drive_pwm -= 1
					if drive_pwm < 0:
						drive_pwm = 0
					print 'drive pwm %u' % drive_pwm
				elif event.key == K_SPACE:
					pass # handled in event.type == KEYUP
				elif event.key == K_ESCAPE:
					running = False
					#kill
					automatic = AUTOMATIC_DEFAULT
					steering_pwm = STEERING_PWM_DEFAULT # center
					drive_pwm = DRIVING_PWM_DEFAULT # stop
					continue
				else:
					#kill
					automatic = AUTOMATIC_DEFAULT
					steering_pwm = STEERING_PWM_DEFAULT # center
					drive_pwm = DRIVING_PWM_DEFAULT # stop

				#time.sleep(0.05)
				motor_command = struct.pack("<BBBB", CB_MOTOR_COMMAND, automatic, steering_pwm, drive_pwm)
				send_packet(motor_command)

		# read serial
		data = s.read(20)
		if data:
			parser.put(data)

		for packet in parser:
			header, = struct.unpack("<B", packet[:1])
			if header == BC_TELEMETRY:
				time, left, right, front_left, front_right, front, mc_x, mc_y, mc_dist, mc_angle, steer, steerPwm, speed, speedPwm = struct.unpack("<IiiiiiiiiiBiBi", packet[1:])
				#print("l %3.2f r %3.2f fl %3.2f fr %3.2f f %3.2f" % (left / FIX_DIV, right / FIX_DIV, front_left / FIX_DIV, front_right / FIX_DIV, front / FIX_DIV))
				print("mc(%.2f, %.2f; %.2f, %.2f)\tsteer (%u): %3u drive (%u): %3u\n" % (mc_x / FIX_DIV, mc_y / FIX_DIV, mc_dist / FIX_DIV, mc_angle / FIX_DIV, steer, steerPwm, speed, speedPwm))
				sys.stdout.flush()

if __name__=="__main__":
	run()
<commit_msg>removed change_steering and change_speed fields from the telemetry packet<commit_after>import sys
import time
import datetime
import argparse
import struct

import serial
import pygame
from pygame.locals import *

sys.path.append('../lib')
import hdlc

FIX_DIV = 65536.0

parser = argparse.ArgumentParser(description="RustTelemetry", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--port", type=str, default="/dev/tty.usbserial-A8008iwL", help="usb serial port device eg. /dev/ttyUSB0")
args = parser.parse_args()

s = serial.Serial(args.port, 57600, timeout=0.1);
s.flushInput()
s.flushOutput()

parser = hdlc.HdlcChecksummed()

# header bytes
BC_TELEMETRY = 0x01
CB_MOTOR_COMMAND = 0x02

AUTOMATIC_DEFAULT = 0
STEERING_PWM_DEFAULT = 90
DRIVING_PWM_DEFAULT = 92

def send_packet(data):
	data = hdlc.add_checksum(data)
	data = hdlc.escape_delimit(data)
	s.write(data)

def run():
	pygame.init()
	pygame.display.set_caption("RustTelemetry")

	screen = pygame.display.set_mode((600,480))
	# by default the key repeat is disabled, enable it
	pygame.key.set_repeat(50, 50)

	running = True
	automatic = AUTOMATIC_DEFAULT
	steering_pwm = STEERING_PWM_DEFAULT # center
	drive_pwm = DRIVING_PWM_DEFAULT # stop

	while running:
		for event in pygame.event.get():
			if event.type == pygame.QUIT:
				running = False
			elif event.type == KEYUP:
				if event.key == K_SPACE:
					automatic = 0 if automatic == 1 else 1
					steering_pwm = STEERING_PWM_DEFAULT # center
					drive_pwm = DRIVING_PWM_DEFAULT # stop
			elif event.type == KEYDOWN:
				if event.key == K_RIGHT:
					steering_pwm += 1
					if steering_pwm > 180:
						steering_pwm = 180
					print 'steering pwm %u' % steering_pwm
				elif event.key == K_LEFT:
					steering_pwm -= 1
					if steering_pwm < 0:
						steering_pwm = 0
					print 'steering pwm %u' % steering_pwm
				elif event.key == K_UP:
					drive_pwm += 1
					if drive_pwm > 180:
						drive_pwm = 180
					print 'drive pwm %u' % drive_pwm
				elif event.key == K_DOWN:
					drive_pwm -= 1
					if drive_pwm < 0:
						drive_pwm = 0
					print 'drive pwm %u' % drive_pwm
				elif event.key == K_SPACE:
					pass # handled in event.type == KEYUP
				elif event.key == K_ESCAPE:
					running = False
					#kill
					automatic = AUTOMATIC_DEFAULT
					steering_pwm = STEERING_PWM_DEFAULT # center
					drive_pwm = DRIVING_PWM_DEFAULT # stop
					continue
				else:
					#kill
					automatic = AUTOMATIC_DEFAULT
					steering_pwm = STEERING_PWM_DEFAULT # center
					drive_pwm = DRIVING_PWM_DEFAULT # stop

				#time.sleep(0.05)
				motor_command = struct.pack("<BBBB", CB_MOTOR_COMMAND, automatic, steering_pwm, drive_pwm)
				send_packet(motor_command)

		# read serial
		data = s.read(20)
		if data:
			parser.put(data)

		for packet in parser:
			header, = struct.unpack("<B", packet[:1])
			if header == BC_TELEMETRY:
				time, left, right, front_left, front_right, front, mc_x, mc_y, mc_dist, mc_angle, steerPwm, speedPwm = struct.unpack("<Iiiiiiiiiiii", packet[1:])
				#print("l %3.2f r %3.2f fl %3.2f fr %3.2f f %3.2f" % (left / FIX_DIV, right / FIX_DIV, front_left / FIX_DIV, front_right / FIX_DIV, front / FIX_DIV))
				print("mc(%.2f, %.2f; %.2f, %.2f)\tsteer: %3u drive: %3u\n" % (mc_x / FIX_DIV, mc_y / FIX_DIV, mc_dist / FIX_DIV, mc_angle / FIX_DIV, steerPwm, speedPwm))
				sys.stdout.flush()

if __name__=="__main__":
	run()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

#
# This file is part of LiteX-Boards.
#
# Copyright (c) 2022 Icenowy Zheng <icenowy@aosc.io>
# Copyright (c) 2022 Florent Kermarrec <florent@enjoy-digital.fr>
# SPDX-License-Identifier: BSD-2-Clause

from migen import *
from migen.genlib.resetsync import AsyncResetSynchronizer

from litex.soc.cores.clock.gowin_gw2a import GW2APLL
from litex.soc.integration.soc_core import *
from litex.soc.integration.soc import SoCRegion
from litex.soc.integration.builder import *
from litex.soc.cores.led import LedChaser, WS2812
from litex.soc.cores.gpio import GPIOIn
from litex.soc.cores.video import *

from liteeth.phy.rmii import LiteEthPHYRMII

from litex_boards.platforms import sipeed_tang_primer_20k

from litedram.common import PHYPadsReducer
from litedram.modules import MT41J128M16
from litedram.phy import GW2DDRPHY

# CRG ----------------------------------------------------------------------------------------------

class _CRG(Module):
    def __init__(self, platform, sys_clk_freq, with_video_pll=False):
        self.rst = Signal()
        self.clock_domains.cd_sys     = ClockDomain()
        self.clock_domains.cd_por     = ClockDomain()
        self.clock_domains.cd_init    = ClockDomain()
        self.clock_domains.cd_sys2x   = ClockDomain()
        self.clock_domains.cd_sys2x_i = ClockDomain()

        # # #

        self.stop  = Signal()
        self.reset = Signal()

        # Clk
        clk27 = platform.request("clk27")

        # PLL
        self.submodules.pll = pll = GW2APLL(devicename=platform.devicename, device=platform.device)
        pll.register_clkin(clk27, 27e6)
        pll.create_clkout(self.cd_sys2x_i, 2*sys_clk_freq)
        self.specials += [
            Instance("DHCEN",
                i_CLKIN  = self.cd_sys2x_i.clk,
                i_CE     = self.stop,
                o_CLKOUT = self.cd_sys2x.clk),
            Instance("CLKDIV",
                p_DIV_MODE = "2",
                i_CALIB    = 0,
                i_HCLKIN   = self.cd_sys2x.clk,
                i_RESETN   = ~self.reset,
                o_CLKOUT   = self.cd_sys.clk),
            AsyncResetSynchronizer(self.cd_sys, pll.reset | self.reset),
        ]

        # Power on reset (the onboard POR is not aware of reprogramming)
        por_count = Signal(16, reset=2**16-1)
        por_done  = Signal()
        self.comb += self.cd_por.clk.eq(clk27)
        self.comb += por_done.eq(por_count == 0)
        self.sync.por += If(~por_done, por_count.eq(por_count - 1))
        self.comb += pll.reset.eq(~por_done)

        # Init clock domain
        self.comb += self.cd_init.clk.eq(clk27)
        self.comb += self.cd_init.rst.eq(pll.reset)

        # Video PLL
        if with_video_pll:
            self.submodules.video_pll = video_pll = GW2APLL(devicename=platform.devicename, device=platform.device)
            video_pll.register_clkin(clk27, 27e6)
            self.clock_domains.cd_hdmi   = ClockDomain()
            self.clock_domains.cd_hdmi5x = ClockDomain()
            video_pll.create_clkout(self.cd_hdmi5x, 125e6, margin=1e-3)
            self.specials += Instance("CLKDIV",
                p_DIV_MODE = "5",
                i_RESETN   = 1, # Disable reset signal.
                i_CALIB    = 0, # No calibration.
                i_HCLKIN   = self.cd_hdmi5x.clk,
                o_CLKOUT   = self.cd_hdmi.clk
            )

# BaseSoC ------------------------------------------------------------------------------------------

class BaseSoC(SoCCore):
    def __init__(self, sys_clk_freq=int(48e6),
        with_spi_flash      = False,
        with_led_chaser     = True,
        with_rgb_led        = False,
        with_buttons        = True,
        with_video_terminal = False,
        with_ethernet       = False,
        with_etherbone      = False,
        eth_ip              = "192.168.1.50",
        eth_dynamic_ip      = False,
        dock                = "standard",
        **kwargs):

        assert dock in ["standard", "lite"]

        platform = sipeed_tang_primer_20k.Platform(dock, toolchain="gowin")

        if dock == "lite":
            with_led_chaser = False # No leds on core board nor on dock lite.

        # CRG --------------------------------------------------------------------------------------
        self.submodules.crg = _CRG(platform, sys_clk_freq, with_video_pll=with_video_terminal)

        # SoCCore ----------------------------------------------------------------------------------
        SoCCore.__init__(self, platform, sys_clk_freq, ident="LiteX SoC on Tang Primer 20K", **kwargs)

        # DDR3 SDRAM -------------------------------------------------------------------------------
        # FIXME: WIP.
        if not self.integrated_main_ram_size:
            self.submodules.ddrphy = GW2DDRPHY(
                pads         = PHYPadsReducer(platform.request("ddram"), [0, 1]),
                sys_clk_freq = sys_clk_freq
            )
            self.ddrphy.settings.rtt_nom = "disabled"
            self.comb += self.crg.stop.eq(self.ddrphy.init.stop)
            self.comb += self.crg.reset.eq(self.ddrphy.init.reset)
            self.add_sdram("sdram",
                phy           = self.ddrphy,
                module        = MT41J128M16(sys_clk_freq, "1:2"),
                l2_cache_size = 0
            )
            # ./sipeed_tang_primer_20k.py --cpu-variant=lite --uart-name=crossover+uartbone --csr-csv=csr.csv --build --load
            # litex_server --uart --uart-port=/dev/ttyUSB2
            # litex_term crossover
            # litescope_cli
            if kwargs["uart_name"] == "crossover+uartbone":
                from litescope import LiteScopeAnalyzer
                analyzer_signals = [
                    self.ddrphy.dfi.p0,
                    self.ddrphy.dfi.p0.wrdata_en,
                    self.ddrphy.dfi.p1.rddata_en,
                ]
                self.submodules.analyzer = LiteScopeAnalyzer(analyzer_signals,
                depth        = 128,
                clock_domain = "sys",
                samplerate   = sys_clk_freq,
                csr_csv      = "analyzer.csv"
                )

        # SPI Flash --------------------------------------------------------------------------------
        if with_spi_flash:
            from litespi.modules import W25Q32JV as SpiFlashModule
            from litespi.opcodes import SpiNorFlashOpCodes as Codes
            self.add_spi_flash(mode="1x", module=SpiFlashModule(Codes.READ_1_1_1))

        # Ethernet / Etherbone ---------------------------------------------------------------------
        if with_ethernet or with_etherbone:
            # FIXME: Un-tested.
            from liteeth.phy.rmii import LiteEthPHYRMII
            self.submodules.ethphy = LiteEthPHYRMII(
                clock_pads = self.platform.request("eth_clocks"),
                pads       = self.platform.request("eth"),
                refclk_cd  = None
            )
            if with_ethernet:
                self.add_ethernet(phy=self.ethphy, dynamic_ip=eth_dynamic_ip, with_timing_constraints=False)
            if with_etherbone:
                self.add_etherbone(phy=self.ethphy, ip_address=eth_ip, with_timing_constraints=False)

        # Video ------------------------------------------------------------------------------------
        if with_video_terminal:
            # FIXME: Un-tested.
            hdmi_pads = platform.request("hdmi")
            self.comb += hdmi_pads.hdp.eq(1)
            self.submodules.videophy = VideoHDMIPHY(hdmi_pads, clock_domain="hdmi", pn_swap=["r", "g", "b"])
            self.add_video_colorbars(phy=self.videophy, timings="640x480@60Hz", clock_domain="hdmi")
            #self.add_video_terminal(phy=self.videophy, timings="640x480@75Hz", clock_domain="hdmi")

        # Leds -------------------------------------------------------------------------------------
        if with_led_chaser:
            self.submodules.leds = LedChaser(
                pads         = platform.request_all("led"),
                sys_clk_freq = sys_clk_freq
            )

        # RGB Led ----------------------------------------------------------------------------------
        if with_rgb_led:
            self.submodules.rgb_led = WS2812(
                pad          = platform.request("rgb_led"),
                nleds        = 1,
                sys_clk_freq = sys_clk_freq
            )
            self.bus.add_slave(name="rgb_led", slave=self.rgb_led.bus, region=SoCRegion(
                origin = 0x2000_0000,
                size   = 4,
            ))

        # Buttons ----------------------------------------------------------------------------------
        if with_buttons:
            self.submodules.buttons = GPIOIn(pads=~platform.request_all("btn_n"))

# Build --------------------------------------------------------------------------------------------

def main():
    from litex.soc.integration.soc import LiteXSoCArgumentParser
    parser = LiteXSoCArgumentParser(description="LiteX SoC on Tang Primer 20K")
    target_group = parser.add_argument_group(title="Target options")
    target_group.add_argument("--dock",         default="standard",    help="Dock version (standard (default) or lite.")
    target_group.add_argument("--build",        action="store_true",   help="Build bitstream.")
    target_group.add_argument("--load",         action="store_true",   help="Load bitstream.")
    target_group.add_argument("--flash",        action="store_true",   help="Flash Bitstream.")
    target_group.add_argument("--sys-clk-freq", default=48e6,          help="System clock frequency.")
    sdopts = target_group.add_mutually_exclusive_group()
    sdopts.add_argument("--with-spi-sdcard",      action="store_true", help="Enable SPI-mode SDCard support.")
    sdopts.add_argument("--with-sdcard",          action="store_true", help="Enable SDCard support.")
    target_group.add_argument("--with-spi-flash", action="store_true", help="Enable SPI Flash (MMAPed).")
    target_group.add_argument("--with-video-terminal", action="store_true", help="Enable Video Terminal (HDMI).")
    ethopts = target_group.add_mutually_exclusive_group()
    ethopts.add_argument("--with-ethernet",  action="store_true",    help="Add Ethernet.")
    ethopts.add_argument("--with-etherbone", action="store_true",    help="Add EtherBone.")
    target_group.add_argument("--eth-ip",          default="192.168.1.50", help="Etherbone IP address.")
    target_group.add_argument("--eth-dynamic-ip",  action="store_true",    help="Enable dynamic Ethernet IP addresses setting.")
    builder_args(parser)
    soc_core_args(parser)
    args = parser.parse_args()

    soc = BaseSoC(
        sys_clk_freq        = int(float(args.sys_clk_freq)),
        with_spi_flash      = args.with_spi_flash,
        with_video_terminal = args.with_video_terminal,
        with_ethernet       = args.with_ethernet,
        with_etherbone      = args.with_etherbone,
        eth_ip              = args.eth_ip,
        eth_dynamic_ip      = args.eth_dynamic_ip,
        dock                = args.dock,
        **soc_core_argdict(args)
    )
    if args.with_spi_sdcard:
        soc.add_spi_sdcard()
    if args.with_sdcard:
        soc.add_sdcard()

    builder = Builder(soc, **builder_argdict(args))
    if args.build:
        builder.build()

    if args.load:
        prog = soc.platform.create_programmer()
        prog.load_bitstream(builder.get_bitstream_filename(mode="sram"))

    if args.flash:
        prog = soc.platform.create_programmer()
        prog.flash(0, builder.get_bitstream_filename(mode="flash", ext=".fs"), external=True)

if __name__ == "__main__":
    main()
<commit_msg>sipeed_tang_primer_20k: Cleanup CRG.<commit_after>#!/usr/bin/env python3

#
# This file is part of LiteX-Boards.
#
# Copyright (c) 2022 Icenowy Zheng <icenowy@aosc.io>
# Copyright (c) 2022 Florent Kermarrec <florent@enjoy-digital.fr>
# SPDX-License-Identifier: BSD-2-Clause

from migen import *
from migen.genlib.resetsync import AsyncResetSynchronizer

from litex.soc.cores.clock.gowin_gw2a import GW2APLL
from litex.soc.integration.soc_core import *
from litex.soc.integration.soc import SoCRegion
from litex.soc.integration.builder import *
from litex.soc.cores.led import LedChaser, WS2812
from litex.soc.cores.gpio import GPIOIn
from litex.soc.cores.video import *

from liteeth.phy.rmii import LiteEthPHYRMII

from litex_boards.platforms import sipeed_tang_primer_20k

from litedram.common import PHYPadsReducer
from litedram.modules import MT41J128M16
from litedram.phy import GW2DDRPHY

# CRG ----------------------------------------------------------------------------------------------

class _CRG(Module):
    def __init__(self, platform, sys_clk_freq, with_video_pll=False):
        self.rst = Signal()
        self.clock_domains.cd_sys     = ClockDomain()
        self.clock_domains.cd_por     = ClockDomain()
        self.clock_domains.cd_init    = ClockDomain()
        self.clock_domains.cd_sys2x   = ClockDomain()
        self.clock_domains.cd_sys2x_i = ClockDomain()

        # # #

        self.stop  = Signal()
        self.reset = Signal()

        # Clk
        clk27 = platform.request("clk27")

        # Power on reset (the onboard POR is not aware of reprogramming)
        por_count = Signal(16, reset=2**16-1)
        por_done  = Signal()
        self.comb += self.cd_por.clk.eq(clk27)
        self.comb += por_done.eq(por_count == 0)
        self.sync.por += If(~por_done, por_count.eq(por_count - 1))

        # PLL
        self.submodules.pll = pll = GW2APLL(devicename=platform.devicename, device=platform.device)
        self.comb += pll.reset.eq(~por_done)
        pll.register_clkin(clk27, 27e6)
        pll.create_clkout(self.cd_sys2x_i, 2*sys_clk_freq)
        self.specials += [
            Instance("DHCEN",
                i_CLKIN  = self.cd_sys2x_i.clk,
                i_CE     = self.stop,
                o_CLKOUT = self.cd_sys2x.clk),
            Instance("CLKDIV",
                p_DIV_MODE = "2",
                i_CALIB    = 0,
                i_HCLKIN   = self.cd_sys2x.clk,
                i_RESETN   = ~self.reset,
                o_CLKOUT   = self.cd_sys.clk),
            AsyncResetSynchronizer(self.cd_sys, ~pll.locked | self.reset),
        ]

        # Init clock domain
        self.comb += self.cd_init.clk.eq(clk27)
        self.comb += self.cd_init.rst.eq(pll.reset)

        # Video PLL
        if with_video_pll:
            self.submodules.video_pll = video_pll = GW2APLL(devicename=platform.devicename, device=platform.device)
            video_pll.register_clkin(clk27, 27e6)
            self.clock_domains.cd_hdmi   = ClockDomain()
            self.clock_domains.cd_hdmi5x = ClockDomain()
            video_pll.create_clkout(self.cd_hdmi5x, 125e6, margin=1e-3)
            self.specials += Instance("CLKDIV",
                p_DIV_MODE = "5",
                i_RESETN   = 1, # Disable reset signal.
                i_CALIB    = 0, # No calibration.
                i_HCLKIN   = self.cd_hdmi5x.clk,
                o_CLKOUT   = self.cd_hdmi.clk
            )

# BaseSoC ------------------------------------------------------------------------------------------

class BaseSoC(SoCCore):
    def __init__(self, sys_clk_freq=int(48e6),
        with_spi_flash      = False,
        with_led_chaser     = True,
        with_rgb_led        = False,
        with_buttons        = True,
        with_video_terminal = False,
        with_ethernet       = False,
        with_etherbone      = False,
        eth_ip              = "192.168.1.50",
        eth_dynamic_ip      = False,
        dock                = "standard",
        **kwargs):

        assert dock in ["standard", "lite"]

        platform = sipeed_tang_primer_20k.Platform(dock, toolchain="gowin")

        if dock == "lite":
            with_led_chaser = False # No leds on core board nor on dock lite.

        # CRG --------------------------------------------------------------------------------------
        self.submodules.crg = _CRG(platform, sys_clk_freq, with_video_pll=with_video_terminal)

        # SoCCore ----------------------------------------------------------------------------------
        SoCCore.__init__(self, platform, sys_clk_freq, ident="LiteX SoC on Tang Primer 20K", **kwargs)

        # DDR3 SDRAM -------------------------------------------------------------------------------
        # FIXME: WIP.
        if not self.integrated_main_ram_size:
            self.submodules.ddrphy = GW2DDRPHY(
                pads         = PHYPadsReducer(platform.request("ddram"), [0, 1]),
                sys_clk_freq = sys_clk_freq
            )
            self.ddrphy.settings.rtt_nom = "disabled"
            self.comb += self.crg.stop.eq(self.ddrphy.init.stop)
            self.comb += self.crg.reset.eq(self.ddrphy.init.reset)
            self.add_sdram("sdram",
                phy           = self.ddrphy,
                module        = MT41J128M16(sys_clk_freq, "1:2"),
                l2_cache_size = 0
            )
            # ./sipeed_tang_primer_20k.py --cpu-variant=lite --uart-name=crossover+uartbone --csr-csv=csr.csv --build --load
            # litex_server --uart --uart-port=/dev/ttyUSB2
            # litex_term crossover
            # litescope_cli
            if kwargs["uart_name"] == "crossover+uartbone":
                from litescope import LiteScopeAnalyzer
                analyzer_signals = [
                    self.ddrphy.dfi.p0,
                    self.ddrphy.dfi.p0.wrdata_en,
                    self.ddrphy.dfi.p1.rddata_en,
                ]
                self.submodules.analyzer = LiteScopeAnalyzer(analyzer_signals,
                depth        = 128,
                clock_domain = "sys",
                samplerate   = sys_clk_freq,
                csr_csv      = "analyzer.csv"
                )

        # SPI Flash --------------------------------------------------------------------------------
        if with_spi_flash:
            from litespi.modules import W25Q32JV as SpiFlashModule
            from litespi.opcodes import SpiNorFlashOpCodes as Codes
            self.add_spi_flash(mode="1x", module=SpiFlashModule(Codes.READ_1_1_1))

        # Ethernet / Etherbone ---------------------------------------------------------------------
        if with_ethernet or with_etherbone:
            # FIXME: Un-tested.
            from liteeth.phy.rmii import LiteEthPHYRMII
            self.submodules.ethphy = LiteEthPHYRMII(
                clock_pads = self.platform.request("eth_clocks"),
                pads       = self.platform.request("eth"),
                refclk_cd  = None
            )
            if with_ethernet:
                self.add_ethernet(phy=self.ethphy, dynamic_ip=eth_dynamic_ip, with_timing_constraints=False)
            if with_etherbone:
                self.add_etherbone(phy=self.ethphy, ip_address=eth_ip, with_timing_constraints=False)

        # Video ------------------------------------------------------------------------------------
        if with_video_terminal:
            # FIXME: Un-tested.
            hdmi_pads = platform.request("hdmi")
            self.comb += hdmi_pads.hdp.eq(1)
            self.submodules.videophy = VideoHDMIPHY(hdmi_pads, clock_domain="hdmi", pn_swap=["r", "g", "b"])
            self.add_video_colorbars(phy=self.videophy, timings="640x480@60Hz", clock_domain="hdmi")
            #self.add_video_terminal(phy=self.videophy, timings="640x480@75Hz", clock_domain="hdmi")

        # Leds -------------------------------------------------------------------------------------
        if with_led_chaser:
            self.submodules.leds = LedChaser(
                pads         = platform.request_all("led"),
                sys_clk_freq = sys_clk_freq
            )

        # RGB Led ----------------------------------------------------------------------------------
        if with_rgb_led:
            self.submodules.rgb_led = WS2812(
                pad          = platform.request("rgb_led"),
                nleds        = 1,
                sys_clk_freq = sys_clk_freq
            )
            self.bus.add_slave(name="rgb_led", slave=self.rgb_led.bus, region=SoCRegion(
                origin = 0x2000_0000,
                size   = 4,
            ))

        # Buttons ----------------------------------------------------------------------------------
        if with_buttons:
            self.submodules.buttons = GPIOIn(pads=~platform.request_all("btn_n"))

# Build --------------------------------------------------------------------------------------------

def main():
    from litex.soc.integration.soc import LiteXSoCArgumentParser
    parser = LiteXSoCArgumentParser(description="LiteX SoC on Tang Primer 20K")
    target_group = parser.add_argument_group(title="Target options")
    target_group.add_argument("--dock",         default="standard",    help="Dock version (standard (default) or lite.")
    target_group.add_argument("--build",        action="store_true",   help="Build bitstream.")
    target_group.add_argument("--load",         action="store_true",   help="Load bitstream.")
    target_group.add_argument("--flash",        action="store_true",   help="Flash Bitstream.")
    target_group.add_argument("--sys-clk-freq", default=48e6,          help="System clock frequency.")
    sdopts = target_group.add_mutually_exclusive_group()
    sdopts.add_argument("--with-spi-sdcard",      action="store_true", help="Enable SPI-mode SDCard support.")
    sdopts.add_argument("--with-sdcard",          action="store_true", help="Enable SDCard support.")
    target_group.add_argument("--with-spi-flash", action="store_true", help="Enable SPI Flash (MMAPed).")
    target_group.add_argument("--with-video-terminal", action="store_true", help="Enable Video Terminal (HDMI).")
    ethopts = target_group.add_mutually_exclusive_group()
    ethopts.add_argument("--with-ethernet",  action="store_true",    help="Add Ethernet.")
    ethopts.add_argument("--with-etherbone", action="store_true",    help="Add EtherBone.")
    target_group.add_argument("--eth-ip",          default="192.168.1.50", help="Etherbone IP address.")
    target_group.add_argument("--eth-dynamic-ip",  action="store_true",    help="Enable dynamic Ethernet IP addresses setting.")
    builder_args(parser)
    soc_core_args(parser)
    args = parser.parse_args()

    soc = BaseSoC(
        sys_clk_freq        = int(float(args.sys_clk_freq)),
        with_spi_flash      = args.with_spi_flash,
        with_video_terminal = args.with_video_terminal,
        with_ethernet       = args.with_ethernet,
        with_etherbone      = args.with_etherbone,
        eth_ip              = args.eth_ip,
        eth_dynamic_ip      = args.eth_dynamic_ip,
        dock                = args.dock,
        **soc_core_argdict(args)
    )
    if args.with_spi_sdcard:
        soc.add_spi_sdcard()
    if args.with_sdcard:
        soc.add_sdcard()

    builder = Builder(soc, **builder_argdict(args))
    if args.build:
        builder.build()

    if args.load:
        prog = soc.platform.create_programmer()
        prog.load_bitstream(builder.get_bitstream_filename(mode="sram"))

    if args.flash:
        prog = soc.platform.create_programmer()
        prog.flash(0, builder.get_bitstream_filename(mode="flash", ext=".fs"), external=True)

if __name__ == "__main__":
    main()
<|endoftext|>