From 356264fa9f566e8edecbcbd4f7a7952e5a161c9d Mon Sep 17 00:00:00 2001 From: yonishelach Date: Wed, 8 Mar 2023 11:01:28 +0200 Subject: [PATCH 01/11] [Build] Fix html links, Add .html as source in documentation --- cli/marketplace/build.py | 116 ++++++++++++++++++++++++++------------- 1 file changed, 78 insertions(+), 38 deletions(-) diff --git a/cli/marketplace/build.py b/cli/marketplace/build.py index 1a6c56b76..bc6d91293 100644 --- a/cli/marketplace/build.py +++ b/cli/marketplace/build.py @@ -18,7 +18,7 @@ import subprocess import uuid from pathlib import Path -from typing import Union, Optional, Set, Dict, List +from typing import Dict, List, Optional, Set, Union import click import yaml @@ -26,13 +26,8 @@ from sphinx.cmd.build import main as sphinx_build_cmd from sphinx.ext.apidoc import main as sphinx_apidoc_cmd -from cli.helpers import ( - is_item_dir, - render_jinja, - PROJECT_ROOT, - get_item_yaml_values, - get_mock_requirements, -) +from cli.helpers import (PROJECT_ROOT, get_item_yaml_values, + get_mock_requirements, is_item_dir, render_jinja) from cli.marketplace.changelog import ChangeLog from cli.path_iterator import PathIterator @@ -68,6 +63,14 @@ default=False, help="When this flag is set, the process will output extra information", ) +@click.option( + "-f", + "--force-update", + "force_update_items", + is_flag=True, + default=False, + help="When this flag is set, item pages will be created even if the item did not changed", +) def build_marketplace_cli( source_dir: str, source_name: str, @@ -75,6 +78,7 @@ def build_marketplace_cli( temp_dir: str, channel: str, verbose: bool, + force_update_items: bool, ): build_marketplace( source_dir, @@ -83,6 +87,7 @@ def build_marketplace_cli( temp_dir, channel, verbose, + force_update_items, ) @@ -93,6 +98,7 @@ def build_marketplace( temp_dir: str = "/tmp", channel: str = "development", verbose: bool = False, + force_update_items: bool = False, ): """Main entry point to marketplace building @@ -103,6 +109,8 @@ def build_marketplace( if not provided '/tmp/' will be used :param channel: The name of the marketplace channel to write to :param verbose: When True, additional debug information will be written to stdout + :param force_update_items: If True, items will be updated unrelated if they are not changed. + The purpose of this flag is to fix existed broken pages (e.g. broken links) """ global _verbose _verbose = verbose @@ -152,9 +160,15 @@ def build_marketplace( render_html_files(temp_docs) change_log = ChangeLog() - copy_static_resources(marketplace_dir, temp_docs) + copy_resources(marketplace_dir, temp_docs) - update_or_create_items(source_dir, marketplace_dir, temp_docs, change_log) + update_or_create_items( + source_dir, + marketplace_dir, + temp_docs, + change_log, + force_update=force_update_items, + ) build_catalog_json( marketplace_dir=marketplace_dir, catalog_path=(marketplace_root / "catalog.json"), @@ -212,18 +226,22 @@ def write_index_html(marketplace_root: Union[str, Path]): shutil.copy(template_path, index_path) -def copy_static_resources(marketplace_dir, temp_docs): +def copy_resources(marketplace_dir, temp_docs): marketplace_static = marketplace_dir / "_static" - if not marketplace_static.exists(): - click.echo("Copying static resources...") - shutil.copytree(temp_docs / "_build/_static", marketplace_static) - shutil.copytree(temp_docs / "_build/_modules", marketplace_dir / "_modules") + click.echo("Copying static resources...") + shutil.copytree( + temp_docs / "_build/_static", marketplace_static, dirs_exist_ok=True + ) -def update_or_create_items(source_dir, marketplace_dir, temp_docs, change_log): +def update_or_create_items( + source_dir, marketplace_dir, temp_docs, change_log, force_update: bool = False +): click.echo("Creating items...") for item_dir in PathIterator(root=source_dir, rule=is_item_dir, as_path=True): - update_or_create_item(item_dir, marketplace_dir, temp_docs, change_log) + update_or_create_item( + item_dir, marketplace_dir, temp_docs, change_log, force_update + ) def build_catalog_json( @@ -329,17 +347,22 @@ def add_assets(item_yaml: dict): def update_or_create_item( - item_dir: Path, marketplace_dir: Path, temp_docs: Path, change_log: ChangeLog + item_dir: Path, + marketplace_dir: Path, + temp_docs: Path, + change_log: ChangeLog, + force_update: bool = False, ): # Copy source directories to target directories, if target already has the directory, archive previous version item_yaml = yaml.full_load(open(item_dir / "item.yaml", "r")) source_version = item_yaml["version"] + relative_path = "../../../" marketplace_item = marketplace_dir / item_dir.stem target_latest = marketplace_item / "latest" target_version = marketplace_item / source_version - if target_version.exists(): + if target_version.exists() and not force_update: latest_item_yaml = yaml.full_load( open(target_latest / "src" / "item.yaml", "r") ) @@ -351,16 +374,21 @@ def update_or_create_item( example_html_name = f"{item_dir.stem}_example.html" build_path = temp_docs / "_build" - source_html = marketplace_dir / "_modules" / item_dir.stem / f"{item_dir.stem}.html" - update_html_resource_paths(source_html, relative_path="../") + source_html = ( + temp_docs / "_build" / "_modules" / item_dir.stem / f"{item_dir.stem}.html" + ) + update_html_resource_paths(source_html, relative_path=relative_path) documentation_html = build_path / documentation_html_name update_html_resource_paths( - documentation_html, relative_path="../../../", with_download=False, item_name=item_dir.stem + documentation_html, + relative_path=relative_path, + with_download=False, + item_name=item_dir.stem, ) example_html = build_path / example_html_name - update_html_resource_paths(example_html, relative_path="../../../") + update_html_resource_paths(example_html, relative_path=relative_path) latest_src = target_latest / "src" version_src = target_version / "src" @@ -445,34 +473,38 @@ def update_or_create_item( def update_html_resource_paths( - html_path: Path, relative_path: str, with_download: bool = True, item_name: str = None + html_path: Path, + relative_path: str, + with_download: bool = True, + item_name: str = None, ): if html_path.exists(): with open(html_path, "r", encoding="utf8") as html: parsed = BeautifulSoup(html.read(), features="html.parser") # Update back to docs link (from source page) - back_to_docs_nodes = parsed.find_all(lambda node: "viewcode-back" in node.get("class", "")) - pattern = r"^.*?(?=.html)" + back_to_docs_nodes = parsed.find_all( + lambda node: "viewcode-back" in node.get("class", "") + ) + pattern = r"^.*?(?={})" for node in back_to_docs_nodes: - node["href"] = re.sub(pattern, "documentation", node["href"]) + node["href"] = re.sub( + pattern.format(".html"), "documentation", node["href"] + ) - # Remove _static from links and replace with src + # Fix links with relative paths: nodes = parsed.find_all( - lambda node: node.name == "link" and "_static" in node.get("href", "") + lambda node: "_static" in node.get("src", "") + or "_static" in node.get("href", "") ) for node in nodes: - node["href"] = f"{relative_path}{node['href']}" + key = "href" if "_static" in node.get("href", "") else "src" + node[key] = re.sub(pattern.format("_static"), relative_path, node[key]) - nodes = parsed.find_all( - lambda node: node.name == "script" - and node.get("src", "").startswith("_static") - ) - for node in nodes: - node["src"] = f"{relative_path}{node['src']}" if with_download: nodes = parsed.find_all(lambda node: "_sources" in node.get("href", "")) for node in nodes: + # fix path and remove example from name: node[ "href" ] = f'../{node["href"].replace("_sources", "src").replace("_example", "")}' @@ -487,7 +519,9 @@ def update_html_resource_paths( # Fix links in source page: if item_name: - nodes = parsed.find_all(lambda node: node.name == "a" and "_modules" in node.get("href", "")) + nodes = parsed.find_all( + lambda node: node.name == "a" and "_modules" in node.get("href", "") + ) for node in nodes: node["href"] = node["href"].replace(f"_modules/{item_name}/", "") @@ -653,4 +687,10 @@ def build_temp_docs(temp_root, temp_docs): if __name__ == "__main__": # build_marketplace_cli() - build_marketplace("../../", "../../../marketp", verbose=True) + build_marketplace( + source_dir="../../../functions", + marketplace_dir="../../../marketplace", + verbose=True, + channel="development", + force_update_items=True, + ) From 2202cafb5adb8fcc8cc93bb21627143a7df494fa Mon Sep 17 00:00:00 2001 From: yonishelach Date: Wed, 19 Apr 2023 09:36:25 +0300 Subject: [PATCH 02/11] Update CI temporarily and update index --- .github/workflows/test-all.yaml | 2 +- cli/marketplace/index.html | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index e946b11b3..d0eb03579 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -167,7 +167,7 @@ jobs: pwd git pull origin cd .. - python functions/functions.py build-marketplace -s functions -m marketplace -c $CHANNEL -v + python functions/functions.py build-marketplace -s functions -m marketplace -c $CHANNEL -v -f - name: Publish marketplace release env: GITHUB_TOKEN: ${{ secrets.MARKETPLACE_ACCESS_TOKEN_V3 }} diff --git a/cli/marketplace/index.html b/cli/marketplace/index.html index b6e38cfa3..d1030e92f 100644 --- a/cli/marketplace/index.html +++ b/cli/marketplace/index.html @@ -115,6 +115,7 @@

Function Hub

item.example = item.example ? `${base_url}/static/example.html` : null; item.functionPath = `${base_url}/static/function.html`; item.itemPath = `${base_url}/static/item.html`; + item.code = `${base_url}/static/${item.rawName}.html`; table.addRow(item); } @@ -189,6 +190,18 @@

Function Hub

}, }, width: 150 + }, + { + title: "Source Code", + field: "code", + headerSort: false, + formatter: "link", + formatterParams: { + label: (cell) => { + return (cell._cell.value === undefined ? '': 'View'); + }, + }, + width: 150 }, { title: "Deployment", From 8b17c8f3dea43adcde9f9a389f16564eac082dca Mon Sep 17 00:00:00 2001 From: yonishelach Date: Wed, 19 Apr 2023 10:20:07 +0300 Subject: [PATCH 03/11] [XGB-Custom] Fix test artifact key name --- xgb_custom/test_xgb_custom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xgb_custom/test_xgb_custom.py b/xgb_custom/test_xgb_custom.py index 6cdada74b..81b77a2e0 100644 --- a/xgb_custom/test_xgb_custom.py +++ b/xgb_custom/test_xgb_custom.py @@ -41,7 +41,7 @@ def test_local_xgb_custom(): "verbose_eval": False, "XGB_max_depth": 2, "XGB_subsample": 0.9, - "test_set_key": "./artifacts/inputs/test-set", + "test_set_key": "test-set", }, inputs={"dataset": run.artifact('xgb-outs').url}, handler="fit", From a469dcaebeea7cb8874fc05eb18b9728019cde84 Mon Sep 17 00:00:00 2001 From: yonishelach Date: Wed, 19 Apr 2023 13:43:33 +0300 Subject: [PATCH 04/11] [XGB-Serving][XGB-Test][XGB-Trainer] Fix tests - artifact key --- xgb_serving/test_xgb_serving.py | 3 +-- xgb_test/test_xgb_test.py | 2 -- xgb_trainer/test_xgb_trainer.py | 7 +++---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/xgb_serving/test_xgb_serving.py b/xgb_serving/test_xgb_serving.py index 9f4dd8244..ce5e8aaa8 100644 --- a/xgb_serving/test_xgb_serving.py +++ b/xgb_serving/test_xgb_serving.py @@ -37,8 +37,7 @@ def test_local_xgb_serving(): "CLASS_objective": "binary:logistic", "CLASS_booster": "gbtree", "FIT_verbose": 0, - "label_column": "labels", - "test_set": "./"}, + "label_column": "labels"}, local=True, inputs={"dataset": gen_data_run.artifact('classifier-data').url}, artifact_path='./') diff --git a/xgb_test/test_xgb_test.py b/xgb_test/test_xgb_test.py index 71dfee3e2..e56b9db01 100644 --- a/xgb_test/test_xgb_test.py +++ b/xgb_test/test_xgb_test.py @@ -51,7 +51,6 @@ def xgb_trainer(): "CLASS_booster": "gbtree", "FIT_verbose": 0, "label_column": "labels", - "test_set": "./artifacts/test-set", }, local=True, inputs={"dataset": data}, @@ -111,7 +110,6 @@ def test_local_xgb_test_import_local_function(): "CLASS_booster": "gbtree", "FIT_verbose": 0, "label_column": "labels", - "test_set": "./artifacts/test-set", }, local=True, inputs={"dataset": data}, diff --git a/xgb_trainer/test_xgb_trainer.py b/xgb_trainer/test_xgb_trainer.py index 1356f72c7..52df8db48 100644 --- a/xgb_trainer/test_xgb_trainer.py +++ b/xgb_trainer/test_xgb_trainer.py @@ -29,6 +29,7 @@ def get_class_data(): 'file_ext': 'csv'}, local=True, artifact_path='./') return run + def test_xgb_trainer_code_to_function(): gen_data_run = get_class_data() fn = code_to_function(name='test_xgb_trainer', @@ -41,8 +42,7 @@ def test_xgb_trainer_code_to_function(): 'CLASS_objective': 'binary:logistic', 'CLASS_booster': 'gbtree', 'FIT_verbose': 0, - 'label_column': 'labels', - 'test_set': './'}, + 'label_column': 'labels'}, local=False, inputs={'dataset': gen_data_run.artifact('classifier-data').url}) @@ -59,8 +59,7 @@ def test_local_xgb_trainer_import_function(): 'CLASS_objective': 'binary:logistic', 'CLASS_booster': 'gbtree', 'FIT_verbose': 0, - 'label_column': 'labels', - 'test_set': './'}, + 'label_column': 'labels'}, local=True, inputs={'dataset': gen_data_run.artifact('classifier-data').url}) assert (run.artifact('model')) \ No newline at end of file From 3301415200e52326bade1e17f99cb6b6d3880860 Mon Sep 17 00:00:00 2001 From: Yoni Shelach <92271540+yonishelach@users.noreply.github.com> Date: Wed, 19 Apr 2023 19:05:40 +0300 Subject: [PATCH 05/11] [Build] Install python 3.9 when testing (#618) --- cli/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/helpers.py b/cli/helpers.py index 64d5505c5..75661f345 100644 --- a/cli/helpers.py +++ b/cli/helpers.py @@ -65,7 +65,7 @@ def install_pipenv(): def install_python(directory: Union[str, Path]): print(f"Installing python for {directory}...") python_install: subprocess.CompletedProcess = subprocess.run( - f"pipenv --rm;pipenv --python 3.7", + f"pipenv --rm;pipenv --python 3.9.13", stdout=sys.stdout, stderr=subprocess.PIPE, cwd=directory, From 0cd1f1585a618c253f201b6f5a63502cdbddb591 Mon Sep 17 00:00:00 2001 From: Yoni Shelach <92271540+yonishelach@users.noreply.github.com> Date: Wed, 19 Apr 2023 19:41:19 +0300 Subject: [PATCH 06/11] [Build] Update python version in CI (#620) * [Build] Install python 3.9 when testing * [Build] Update python version in CI * . --- .github/workflows/ci.yaml | 2 +- .github/workflows/test-all.yaml | 6 +++--- cli/helpers.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 155634dfc..1ab67ffc7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,7 +107,7 @@ jobs: - name: Install python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.9 # Install dependencies - uses: actions/cache@v1 id: cache diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index d0eb03579..fe7248bcd 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -70,7 +70,7 @@ jobs: - name: Install python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.9 # Install dependencies - uses: actions/cache@v1 id: cache @@ -106,7 +106,7 @@ jobs: # - name: Install python # uses: actions/setup-python@v2 # with: -# python-version: 3.7 +# python-version: 3.9 # # Install dependencies # - uses: actions/cache@v1 # id: cache @@ -153,7 +153,7 @@ jobs: - name: Install python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.9 - name: Install requirements run: | cd functions diff --git a/cli/helpers.py b/cli/helpers.py index 75661f345..b44ebc92e 100644 --- a/cli/helpers.py +++ b/cli/helpers.py @@ -65,7 +65,7 @@ def install_pipenv(): def install_python(directory: Union[str, Path]): print(f"Installing python for {directory}...") python_install: subprocess.CompletedProcess = subprocess.run( - f"pipenv --rm;pipenv --python 3.9.13", + f"pipenv --rm;pipenv --python 3.9", stdout=sys.stdout, stderr=subprocess.PIPE, cwd=directory, From 33e7ab8c43579b8609ed4f9654cc7b0d0f06671a Mon Sep 17 00:00:00 2001 From: Yoni Shelach <92271540+yonishelach@users.noreply.github.com> Date: Wed, 19 Apr 2023 19:47:50 +0300 Subject: [PATCH 07/11] Revert "[Build] Update python version in CI (#620)" (#621) This reverts commit 0cd1f1585a618c253f201b6f5a63502cdbddb591. --- .github/workflows/ci.yaml | 2 +- .github/workflows/test-all.yaml | 6 +++--- cli/helpers.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1ab67ffc7..155634dfc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,7 +107,7 @@ jobs: - name: Install python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.7 # Install dependencies - uses: actions/cache@v1 id: cache diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index fe7248bcd..d0eb03579 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -70,7 +70,7 @@ jobs: - name: Install python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.7 # Install dependencies - uses: actions/cache@v1 id: cache @@ -106,7 +106,7 @@ jobs: # - name: Install python # uses: actions/setup-python@v2 # with: -# python-version: 3.9 +# python-version: 3.7 # # Install dependencies # - uses: actions/cache@v1 # id: cache @@ -153,7 +153,7 @@ jobs: - name: Install python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.7 - name: Install requirements run: | cd functions diff --git a/cli/helpers.py b/cli/helpers.py index b44ebc92e..75661f345 100644 --- a/cli/helpers.py +++ b/cli/helpers.py @@ -65,7 +65,7 @@ def install_pipenv(): def install_python(directory: Union[str, Path]): print(f"Installing python for {directory}...") python_install: subprocess.CompletedProcess = subprocess.run( - f"pipenv --rm;pipenv --python 3.9", + f"pipenv --rm;pipenv --python 3.9.13", stdout=sys.stdout, stderr=subprocess.PIPE, cwd=directory, From 7a7473b8f41e80032f381d927214a9076a4a55b8 Mon Sep 17 00:00:00 2001 From: Yoni Shelach <92271540+yonishelach@users.noreply.github.com> Date: Wed, 19 Apr 2023 19:48:09 +0300 Subject: [PATCH 08/11] Revert "[Build] Install python 3.9 when testing (#618)" (#619) This reverts commit 3301415200e52326bade1e17f99cb6b6d3880860. --- cli/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/helpers.py b/cli/helpers.py index 75661f345..64d5505c5 100644 --- a/cli/helpers.py +++ b/cli/helpers.py @@ -65,7 +65,7 @@ def install_pipenv(): def install_python(directory: Union[str, Path]): print(f"Installing python for {directory}...") python_install: subprocess.CompletedProcess = subprocess.run( - f"pipenv --rm;pipenv --python 3.9.13", + f"pipenv --rm;pipenv --python 3.7", stdout=sys.stdout, stderr=subprocess.PIPE, cwd=directory, From 81437da88e99ed48a9e1b0b0b367c4c02db80140 Mon Sep 17 00:00:00 2001 From: Yoni Shelach <92271540+yonishelach@users.noreply.github.com> Date: Wed, 19 Apr 2023 20:26:27 +0300 Subject: [PATCH 09/11] [Build] Build with python 3.9 (#622) * [Build] Build with python 3.9 * . --- .github/workflows/ci.yaml | 5 +++++ .github/workflows/test-all.yaml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 155634dfc..5b4bfcd79 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -108,6 +108,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.7 + # Install python 3.9 + - name: Install python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 # Install dependencies - uses: actions/cache@v1 id: cache diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index d0eb03579..5eff03b0f 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -150,10 +150,10 @@ jobs: with: repository: mlrun/marketplace path: marketplace - - name: Install python + - name: Install python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.9 - name: Install requirements run: | cd functions From 17b6eb054b9542ccc72c2b9a618f01bfcf4393f0 Mon Sep 17 00:00:00 2001 From: guy1992l Date: Wed, 10 Jan 2024 18:01:47 +0200 Subject: [PATCH 10/11] fix1 --- silero_vad/silero_vad.py | 80 ++++++++++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/silero_vad/silero_vad.py b/silero_vad/silero_vad.py index a7a44e055..08ff585fa 100644 --- a/silero_vad/silero_vad.py +++ b/silero_vad/silero_vad.py @@ -66,6 +66,14 @@ def get_result(self) -> Tuple[str, list]: """ return self._audio_file.name, self._result + def to_tuple(self) -> Tuple[str, dict]: + """ + Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). + + :returns: The converted task. + """ + return self.__class__.__name__, {"audio_file": self._audio_file} + class SpeechDiarizationTask(BaseTask): """ @@ -105,12 +113,27 @@ def do_task(self, speech_timestamps: List[List[Dict[str, int]]]): speech_diarization.sort() self._result = speech_diarization + def to_tuple(self) -> Tuple[str, dict]: + """ + Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). + + :returns: The converted task. + """ + task_class, task_kwargs = super().to_tuple() + return task_class, {**task_kwargs, "speaker_labels": self._speaker_labels} + class TaskCreator: """ A task creator to create different tasks to run after the VAD. """ + #: A map from task class name to task class to use in `from_tuple`: + _MAP = { + BaseTask.__name__: BaseTask, + SpeechDiarizationTask.__name__: SpeechDiarizationTask, + } + def __init__(self, task_type: Type[BaseTask], task_kwargs: dict = None): """ Initialize the task creator. @@ -120,7 +143,7 @@ def __init__(self, task_type: Type[BaseTask], task_kwargs: dict = None): self._task_type = task_type self._task_kwargs = task_kwargs or {} - def create_task(self, audio_file: Path): + def create_task(self, audio_file: Path) -> BaseTask: """ Create a task with the given audio file. @@ -130,6 +153,18 @@ def create_task(self, audio_file: Path): """ return self._task_type(audio_file=audio_file, **self._task_kwargs) + @classmethod + def from_tuple(cls, task_tuple: Tuple[str, dict]) -> BaseTask: + """ + Create a task from a tuple of the audio file name and the task kwargs. + + :param task_tuple: The task tuple to create the task from. + + :returns: The created task. + """ + task_class, task_kwargs = task_tuple + return cls._MAP[task_class](**task_kwargs) + class VoiceActivityDetector: """ @@ -196,14 +231,16 @@ def __init__( self._model: torch.Module = None self._get_speech_timestamps: FunctionType = None - def load(self): + def load(self, force_reload: bool = True): """ Load the VAD model. + + :param force_reload: Whether to force reload the model even if it was already loaded. Default is True. """ model, utils = torch.hub.load( repo_or_dir="snakers4/silero-vad", model="silero_vad", - force_reload=True, + force_reload=force_reload, onnx=self._use_onnx, force_onnx_cpu=self._force_onnx_cpu, ) @@ -317,31 +354,40 @@ def _multiprocessing_complete_tasks( """ # Initialize and load the VAD: vad = VoiceActivityDetector(**vad_init_kwargs) - vad.load() + vad.load(force_reload=False) # Start listening to the tasks queue: while True: # Get the task: - task: BaseTask = tasks_queue.get() + task: Tuple[str, dict] = tasks_queue.get() if task == _MULTIPROCESSING_STOP_MARK: break try: + # Create the task: + task = TaskCreator.from_tuple(task_tuple=task) # Run the file through the VAD: speech_timestamps = vad.detect_voice(audio_file=task.audio_file) # Complete the task: task.do_task(speech_timestamps=speech_timestamps) - # Collect the result: - results_queue.put((False, task.get_result())) + # Build the result: + result = (False, task.get_result()) except Exception as exception: - # Collect the error: - results_queue.put((True, (task.audio_file.name, str(exception)))) + # Build the error: + result = (True, (task.audio_file.name, str(exception))) + # Collect the result / error: + results_queue.put(result) # Mark the end of the tasks: results_queue.put(_MULTIPROCESSING_STOP_MARK) # Get the global logger: -_LOGGER = logging.getLogger() +try: + import mlrun + + _LOGGER = mlrun.get_or_create_ctx("silero_vad").logger +except ModuleNotFoundError: + _LOGGER = logging.getLogger() def detect_voice( @@ -702,6 +748,14 @@ def _parallel_run( :returns: The collected results. """ + # Load the VAD (download once, and it will be loaded then per process later on): + if verbose: + _LOGGER.info(f"Loading the VAD model.") + vad = VoiceActivityDetector(**vad_init_kwargs) + vad.load() + if verbose: + _LOGGER.info("VAD model loaded.") + # Check the number of workers: if n_workers > len(audio_files): _LOGGER.warning( @@ -728,12 +782,8 @@ def _parallel_run( ] # Start the multiprocessing processes: - if verbose: - _LOGGER.info(f"Loading the VAD model (per process).") for p in task_completion_processes: p.start() - if verbose: - _LOGGER.info("VAD model loaded.") # Put the tasks in the queue (the progress bar is not accurate as it is updating by the queue which has a max size # of 2*n_workers so the progress bar won't be too off - better than nothing): @@ -745,7 +795,7 @@ def _parallel_run( disable=not verbose, ): # Put the task in the queue: - tasks_queue.put(task_creator.create_task(audio_file=audio_file)) + tasks_queue.put(task_creator.create_task(audio_file=audio_file).to_tuple()) # Put the stop marks in the queue: for _ in range(n_workers): From c3a141c9668953c595c14d286b7beede6a0d15a1 Mon Sep 17 00:00:00 2001 From: guy1992l Date: Wed, 10 Jan 2024 18:39:00 +0200 Subject: [PATCH 11/11] fix2 --- silero_vad/function.yaml | 43 +++++++++++++++++++++++++++++------ silero_vad/item.yaml | 2 +- silero_vad/silero_vad.py | 49 ++++++++++++++++++++-------------------- 3 files changed, 61 insertions(+), 33 deletions(-) diff --git a/silero_vad/function.yaml b/silero_vad/function.yaml index 731d280d8..75d1ce0cc 100644 --- a/silero_vad/function.yaml +++ b/silero_vad/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: silero-vad tag: '' - hash: 064d82f265f7a42c937584473e6092dbbfd8ffe8 + hash: bc0ad5572cc391fcdc93baaee48e1ef949a7984d project: '' labels: author: guyl @@ -15,7 +15,7 @@ spec: args: [] image: '' build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyNCBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgbG9nZ2luZwpmcm9tIG11bHRpcHJvY2Vzc2luZyBpbXBvcnQgUHJvY2VzcywgUXVldWUKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwZXMgaW1wb3J0IEZ1bmN0aW9uVHlwZQpmcm9tIHR5cGluZyBpbXBvcnQgRGljdCwgTGlzdCwgVHVwbGUsIFR5cGUsIFVuaW9uCgppbXBvcnQgdG9yY2gKaW1wb3J0IHRvcmNoYXVkaW8KZnJvbSB0cWRtIGltcG9ydCB0cWRtCgoKY2xhc3MgQmFzZVRhc2s6CiAgICAiIiIKICAgIEEgYmFzZSBjbGFzcyBmb3IgYSB0YXNrIHRvIGNvbXBsZXRlIGFmdGVyIFZBRC4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCBhdWRpb19maWxlOiBQYXRoKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSBiYXNlIHRhc2suCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiBUaGUgYXVkaW8gZmlsZSBhc3NpZ25lZCB0byB0aGUgdGFzay4KICAgICAgICAiIiIKICAgICAgICAjIFN0b3JlIHRoZSBhdWRpbyBmaWxlOgogICAgICAgIHNlbGYuX2F1ZGlvX2ZpbGUgPSBhdWRpb19maWxlCgogICAgICAgICMgUHJlcGFyZSB0aGUgcmVzdWx0OgogICAgICAgIHNlbGYuX3Jlc3VsdCA9IE5vbmUKCiAgICBAcHJvcGVydHkKICAgIGRlZiBhdWRpb19maWxlKHNlbGYpIC0+IFBhdGg6CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSBhdWRpbyBmaWxlIG9mIHRoZSB0YXNrLgoKICAgICAgICA6cmV0dXJuczogVGhlIGF1ZGlvIGZpbGUgb2YgdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2F1ZGlvX2ZpbGUKCiAgICBkZWYgZG9fdGFzaygKICAgICAgICBzZWxmLCBzcGVlY2hfdGltZXN0YW1wczogVW5pb25bTGlzdFtEaWN0W3N0ciwgaW50XV0sIExpc3RbTGlzdFtEaWN0W3N0ciwgaW50XV1dXQogICAgKToKICAgICAgICAiIiIKICAgICAgICBEbyB0aGUgdGFzayBvbiB0aGUgZ2l2ZW4gc3BlZWNoIHRpbWVzdGFtcHMuIFRoZSBiYXNlIHRhc2sgd2lsbCBzaW1wbHkgc2F2ZSB0aGUgc3BlZWNoIHRpbWVzdGFtcHMgYXMgdGhlIHJlc3VsdC4KCiAgICAgICAgOnBhcmFtIHNwZWVjaF90aW1lc3RhbXBzOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgdG8gZG8gdGhlIHRhc2sgb24gYXMgb3V0cHV0dGVkIGZyb20gdGhlIFZBRC4KICAgICAgICAiIiIKICAgICAgICBzZWxmLl9yZXN1bHQgPSBzcGVlY2hfdGltZXN0YW1wcwoKICAgIGRlZiBnZXRfcmVzdWx0KHNlbGYpIC0+IFR1cGxlW3N0ciwgbGlzdF06CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSByZXN1bHQgb2YgdGhlIHRhc2suIEEgdHVwbGUgb2YgdGhlIGF1ZGlvIGZpbGUgbmFtZSBhbmQgdGhlIHJlc3VsdC4KCiAgICAgICAgOnJldHVybnM6IFRoZSByZXN1bHQgb2YgdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2F1ZGlvX2ZpbGUubmFtZSwgc2VsZi5fcmVzdWx0CgoKY2xhc3MgU3BlZWNoRGlhcml6YXRpb25UYXNrKEJhc2VUYXNrKToKICAgICIiIgogICAgQSBzcGVlY2ggZGlhcml6YXRpb24gdGFzay4gVGhlIHRhc2sgd2lsbCBkaWFyaXplIHRoZSBWQUQgc3BlZWNoIHRpbWVzdGFtcHMgaW50byBzcGVha2Vycy4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCBhdWRpb19maWxlOiBQYXRoLCBzcGVha2VyX2xhYmVsczogTGlzdFtzdHJdKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSBzcGVlY2ggZGlhcml6YXRpb24gdGFzay4KCiAgICAgICAgOnBhcmFtIGF1ZGlvX2ZpbGU6ICAgICBUaGUgYXVkaW8gZmlsZSBhc3NpZ25lZCB0byB0aGUgdGFzay4KICAgICAgICA6cGFyYW0gc3BlYWtlcl9sYWJlbHM6IFRoZSBzcGVha2VyIGxhYmVscyB0byB1c2UgZm9yIHRoZSBkaWFyaXphdGlvbi4gSWYgbm90IGdpdmVuLCB0aGUgc3BlYWtlcnMgd2lsbCBiZSBuYW1lZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNwZWFrZXJfMCIsICJzcGVha2VyXzEiLCBldGMuCiAgICAgICAgIiIiCiAgICAgICAgc3VwZXIoKS5fX2luaXRfXyhhdWRpb19maWxlPWF1ZGlvX2ZpbGUpCiAgICAgICAgc2VsZi5fc3BlYWtlcl9sYWJlbHMgPSBzcGVha2VyX2xhYmVscwoKICAgIGRlZiBkb190YXNrKHNlbGYsIHNwZWVjaF90aW1lc3RhbXBzOiBMaXN0W0xpc3RbRGljdFtzdHIsIGludF1dXSk6CiAgICAgICAgIiIiCiAgICAgICAgRG8gdGhlIHRhc2sgb24gdGhlIGdpdmVuIHNwZWVjaCB0aW1lc3RhbXBzLiBUaGUgdGFzayB3aWxsIGRpYXJpemUgdGhlIFZBRCBzcGVlY2ggdGltZXN0YW1wcyBpbnRvIHNwZWFrZXJzLgoKICAgICAgICA6cGFyYW0gc3BlZWNoX3RpbWVzdGFtcHM6IFRoZSBzcGVlY2ggdGltZXN0YW1wcyBwZXIgY2hhbm5lbCB0byBkbyB0aGUgdGFzayBvbiBhcyBvdXRwdXR0ZWQgZnJvbSB0aGUgVkFELgogICAgICAgICIiIgogICAgICAgICMgR2V0IHRoZSBzcGVha2VyIGxhYmVscyAoc2V0IGRlZmF1bHQgaWYgbm90IGdpdmVuKToKICAgICAgICBzcGVha2VyX2xhYmVscyA9IHNlbGYuX3NwZWFrZXJfbGFiZWxzIG9yIFsKICAgICAgICAgICAgZiJzcGVha2VyX3tpfSIgZm9yIGkgaW4gcmFuZ2UobGVuKHNwZWVjaF90aW1lc3RhbXBzKSkKICAgICAgICBdCgogICAgICAgICMgRGlhcml6ZSAtIG9yZ2FuaXplIHRoZSBzcGVlY2ggdGltZXN0YW1wcyBpbnRvIGEgc2luZ2xlIGxpc3Qgb2Ygc3BlYWtlcnMgYW5kIHNvcnQgaXQgYnkgc3RhcnQgdGltZToKICAgICAgICBzcGVlY2hfZGlhcml6YXRpb24gPSBbCiAgICAgICAgICAgIChzcGVlY2hfdGltZXN0YW1wWyJzdGFydCJdLCBzcGVlY2hfdGltZXN0YW1wWyJlbmQiXSwgc3BlYWtlcl9sYWJlbCkKICAgICAgICAgICAgZm9yIHNwZWFrZXJfbGFiZWwsIGNoYW5uZWxfc3BlZWNoX3RpbWVzdGFtcHMgaW4gemlwKAogICAgICAgICAgICAgICAgc3BlYWtlcl9sYWJlbHMsIHNwZWVjaF90aW1lc3RhbXBzCiAgICAgICAgICAgICkKICAgICAgICAgICAgZm9yIHNwZWVjaF90aW1lc3RhbXAgaW4gY2hhbm5lbF9zcGVlY2hfdGltZXN0YW1wcwogICAgICAgIF0KICAgICAgICBzcGVlY2hfZGlhcml6YXRpb24uc29ydCgpCiAgICAgICAgc2VsZi5fcmVzdWx0ID0gc3BlZWNoX2RpYXJpemF0aW9uCgoKY2xhc3MgVGFza0NyZWF0b3I6CiAgICAiIiIKICAgIEEgdGFzayBjcmVhdG9yIHRvIGNyZWF0ZSBkaWZmZXJlbnQgdGFza3MgdG8gcnVuIGFmdGVyIHRoZSBWQUQuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgdGFza190eXBlOiBUeXBlW0Jhc2VUYXNrXSwgdGFza19rd2FyZ3M6IGRpY3QgPSBOb25lKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSB0YXNrIGNyZWF0b3IuCiAgICAgICAgOnBhcmFtIHRhc2tfdHlwZTogVGhlIHRhc2sgdHlwZSAtIGEgYEJhc2VUYXNrYCBzdWJjbGFzcy4KICAgICAgICA6cGFyYW0gdGFza19rd2FyZ3M6IEFkZGl0aW9uYWwga2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgdG8gYmUgY3JlYXRlZCB0YXNrcy4KICAgICAgICAiIiIKICAgICAgICBzZWxmLl90YXNrX3R5cGUgPSB0YXNrX3R5cGUKICAgICAgICBzZWxmLl90YXNrX2t3YXJncyA9IHRhc2tfa3dhcmdzIG9yIHt9CgogICAgZGVmIGNyZWF0ZV90YXNrKHNlbGYsIGF1ZGlvX2ZpbGU6IFBhdGgpOgogICAgICAgICIiIgogICAgICAgIENyZWF0ZSBhIHRhc2sgd2l0aCB0aGUgZ2l2ZW4gYXVkaW8gZmlsZS4KCiAgICAgICAgOnBhcmFtIGF1ZGlvX2ZpbGU6IFRoZSBhdWRpbyBmaWxlIHRvIGFzc2lnbiB0byB0aGUgdGFzay4KCiAgICAgICAgOnJldHVybnM6IFRoZSBjcmVhdGVkIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX3Rhc2tfdHlwZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUsICoqc2VsZi5fdGFza19rd2FyZ3MpCgoKY2xhc3MgVm9pY2VBY3Rpdml0eURldGVjdG9yOgogICAgIiIiCiAgICBBIHZvaWNlIGFjdGl2aXR5IGRldGVjdGlvbiB3cmFwcGVyIGZvciB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtIGh0dHBzOi8vZ2l0aHViLmNvbS9zbmFrZXJzNC9zaWxlcm8tdmFkLgogICAgIiIiCgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsCiAgICAgICAgIyBNb2RlbCBsb2FkaW5nIGt3YXJnczoKICAgICAgICB1c2Vfb25ueDogYm9vbCA9IFRydWUsCiAgICAgICAgZm9yY2Vfb25ueF9jcHU6IGJvb2wgPSBUcnVlLAogICAgICAgICMgRGV0ZWN0aW9uIGt3YXJnczoKICAgICAgICB0aHJlc2hvbGQ6IGZsb2F0ID0gMC41LAogICAgICAgIHNhbXBsaW5nX3JhdGU6IGludCA9IDE2XzAwMCwKICAgICAgICBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiBpbnQgPSAyNTAsCiAgICAgICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiBmbG9hdCA9IGZsb2F0KCJpbmYiKSwKICAgICAgICBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogaW50ID0gMTAwLAogICAgICAgIHdpbmRvd19zaXplX3NhbXBsZXM6IGludCA9IDUxMiwKICAgICAgICBzcGVlY2hfcGFkX21zOiBpbnQgPSAzMCwKICAgICAgICByZXR1cm5fc2Vjb25kczogYm9vbCA9IEZhbHNlLAogICAgICAgIHBlcl9jaGFubmVsOiBib29sID0gRmFsc2UsCiAgICApOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgdGhlIHZvaWNlIGFjdGl2aXR5IGRldGVjdG9yLgoKICAgICAgICA6cGFyYW0gdXNlX29ubng6ICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIE9OTlggZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgICAgIDpwYXJhbSBmb3JjZV9vbm54X2NwdTogICAgICAgICAgV2hldGhlciB0byBmb3JjZSBPTk5YIHRvIHVzZSBDUFUgZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgICAgIDpwYXJhbSB0aHJlc2hvbGQ6ICAgICAgICAgICAgICAgU3BlZWNoIHRocmVzaG9sZC4gU2lsZXJvIFZBRCBvdXRwdXRzIHNwZWVjaCBwcm9iYWJpbGl0aWVzIGZvciBlYWNoIGF1ZGlvIGNodW5rLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHJvYmFiaWxpdGllcyBBQk9WRSB0aGlzIHZhbHVlIGFyZSBjb25zaWRlcmVkIGFzIFNQRUVDSC4gSXQgaXMgYmV0dGVyIHRvIHR1bmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoaXMgcGFyYW1ldGVyIGZvciBlYWNoIGRhdGFzZXQgc2VwYXJhdGVseSwgYnV0ICJsYXp5IiAwLjUgaXMgcHJldHR5IGdvb2QgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb3N0IGRhdGFzZXRzLgogICAgICAgIDpwYXJhbSBzYW1wbGluZ19yYXRlOiAgICAgICAgICAgQ3VycmVudGx5LCBzaWxlcm8gVkFEIG1vZGVscyBzdXBwb3J0IDgwMDAgYW5kIDE2MDAwIHNhbXBsZSByYXRlcy4KICAgICAgICA6cGFyYW0gbWluX3NwZWVjaF9kdXJhdGlvbl9tczogIEZpbmFsIHNwZWVjaCBjaHVua3Mgc2hvcnRlciBtaW5fc3BlZWNoX2R1cmF0aW9uX21zIGFyZSB0aHJvd24gb3V0LgogICAgICAgIDpwYXJhbSBtYXhfc3BlZWNoX2R1cmF0aW9uX3M6ICAgTWF4aW11bSBkdXJhdGlvbiBvZiBzcGVlY2ggY2h1bmtzIGluIHNlY29uZHMuIENodW5rcyBsb25nZXIgdGhhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYG1heF9zcGVlY2hfZHVyYXRpb25fc2Agd2lsbCBiZSBzcGxpdCBhdCB0aGUgdGltZXN0YW1wIG9mIHRoZSBsYXN0IHNpbGVuY2UgdGhhdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFzdHMgbW9yZSB0aGFuIDEwMG1zIChpZiBhbnkpLCB0byBwcmV2ZW50IGFnZ3Jlc3NpdmUgY3V0dGluZy4gT3RoZXJ3aXNlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhleSB3aWxsIGJlIHNwbGl0IGFnZ3Jlc3NpdmVseSBqdXN0IGJlZm9yZSBtYXhfc3BlZWNoX2R1cmF0aW9uX3MuCiAgICAgICAgOnBhcmFtIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBJbiB0aGUgZW5kIG9mIGVhY2ggc3BlZWNoIGNodW5rIHdhaXQgZm9yIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zIGJlZm9yZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VwYXJhdGluZyBpdC4KICAgICAgICA6cGFyYW0gd2luZG93X3NpemVfc2FtcGxlczogICAgIEF1ZGlvIGNodW5rcyBvZiB3aW5kb3dfc2l6ZV9zYW1wbGVzIHNpemUgYXJlIGZlZCB0byB0aGUgc2lsZXJvIFZBRCBtb2RlbC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdBUk5JTkchIFNpbGVybyBWQUQgbW9kZWxzIHdlcmUgdHJhaW5lZCB1c2luZyA1MTIsIDEwMjQsIDE1MzYgc2FtcGxlcyBmb3IgMTYwMDAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSByYXRlIGFuZCAyNTYsIDUxMiwgNzY4IHNhbXBsZXMgZm9yIDgwMDAgc2FtcGxlIHJhdGUuIFZhbHVlcyBvdGhlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGVzZSBtYXkgYWZmZWN0IG1vZGVsIHBlcmZvcm1hbmNlIQogICAgICAgIDpwYXJhbSBzcGVlY2hfcGFkX21zOiAgICAgICAgICAgRmluYWwgc3BlZWNoIGNodW5rcyBhcmUgcGFkZGVkIGJ5IHNwZWVjaF9wYWRfbXMgZWFjaCBzaWRlLgogICAgICAgIDpwYXJhbSByZXR1cm5fc2Vjb25kczogICAgICAgICAgV2hldGhlciByZXR1cm4gdGltZXN0YW1wcyBpbiBzZWNvbmRzLiBGYWxzZSBtZWFucyB0byByZXR1cm4gdGltZXN0YW1wcyBpbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2FtcGxlcyAoZGVmYXVsdCAtIEZhbHNlKS4KICAgICAgICA6cGFyYW0gcGVyX2NoYW5uZWw6ICAgICAgICAgICAgIFdoZXRoZXIgdG8gcmV0dXJuIHRpbWVzdGFtcHMgcGVyIGNoYW5uZWwgKGRlZmF1bHQgLSBGYWxzZSkuIFRoaXMgd2lsbCBydW4gVkFECiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvbiBlYWNoIGNoYW5uZWwgc2VwYXJhdGVseSBhbmQgcmV0dXJuIGEgbGlzdCBvZiB0aW1lc3RhbXBzIHBlciBjaGFubmVsLgogICAgICAgICIiIgogICAgICAgICMgU3RvcmUgY29uZmlndXJhdGlvbnM6CiAgICAgICAgc2VsZi5fdXNlX29ubnggPSB1c2Vfb25ueAogICAgICAgIHNlbGYuX2ZvcmNlX29ubnhfY3B1ID0gZm9yY2Vfb25ueF9jcHUKICAgICAgICBzZWxmLl90aHJlc2hvbGQgPSB0aHJlc2hvbGQKICAgICAgICBzZWxmLl9zYW1wbGluZ19yYXRlID0gc2FtcGxpbmdfcmF0ZQogICAgICAgIHNlbGYuX21pbl9zcGVlY2hfZHVyYXRpb25fbXMgPSBtaW5fc3BlZWNoX2R1cmF0aW9uX21zCiAgICAgICAgc2VsZi5fbWF4X3NwZWVjaF9kdXJhdGlvbl9zID0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zCiAgICAgICAgc2VsZi5fbWluX3NpbGVuY2VfZHVyYXRpb25fbXMgPSBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcwogICAgICAgIHNlbGYuX3dpbmRvd19zaXplX3NhbXBsZXMgPSB3aW5kb3dfc2l6ZV9zYW1wbGVzCiAgICAgICAgc2VsZi5fc3BlZWNoX3BhZF9tcyA9IHNwZWVjaF9wYWRfbXMKICAgICAgICBzZWxmLl9yZXR1cm5fc2Vjb25kcyA9IHJldHVybl9zZWNvbmRzCiAgICAgICAgc2VsZi5fcGVyX2NoYW5uZWwgPSBwZXJfY2hhbm5lbAoKICAgICAgICAjIFByZXBhcmUgdGhlIG1vZGVsIHZhcmlhYmxlcwogICAgICAgIHNlbGYuX21vZGVsOiB0b3JjaC5Nb2R1bGUgPSBOb25lCiAgICAgICAgc2VsZi5fZ2V0X3NwZWVjaF90aW1lc3RhbXBzOiBGdW5jdGlvblR5cGUgPSBOb25lCgogICAgZGVmIGxvYWQoc2VsZik6CiAgICAgICAgIiIiCiAgICAgICAgTG9hZCB0aGUgVkFEIG1vZGVsLgogICAgICAgICIiIgogICAgICAgIG1vZGVsLCB1dGlscyA9IHRvcmNoLmh1Yi5sb2FkKAogICAgICAgICAgICByZXBvX29yX2Rpcj0ic25ha2VyczQvc2lsZXJvLXZhZCIsCiAgICAgICAgICAgIG1vZGVsPSJzaWxlcm9fdmFkIiwKICAgICAgICAgICAgZm9yY2VfcmVsb2FkPVRydWUsCiAgICAgICAgICAgIG9ubng9c2VsZi5fdXNlX29ubngsCiAgICAgICAgICAgIGZvcmNlX29ubnhfY3B1PXNlbGYuX2ZvcmNlX29ubnhfY3B1LAogICAgICAgICkKICAgICAgICBzZWxmLl9tb2RlbCA9IG1vZGVsCiAgICAgICAgKAogICAgICAgICAgICBzZWxmLl9nZXRfc3BlZWNoX3RpbWVzdGFtcHMsCiAgICAgICAgICAgIF8sICAjIHNhdmVfYXVkaW8sCiAgICAgICAgICAgIF8sICAjIHJlYWRfYXVkaW8sCiAgICAgICAgICAgIF8sICAjIFZBREl0ZXJhdG9yLAogICAgICAgICAgICBfLCAgIyBjb2xsZWN0X2NodW5rcwogICAgICAgICkgPSB1dGlscwoKICAgIGRlZiBkZXRlY3Rfdm9pY2UoCiAgICAgICAgc2VsZiwKICAgICAgICBhdWRpb19maWxlOiBQYXRoLAogICAgKSAtPiBVbmlvbltMaXN0W0RpY3Rbc3RyLCBpbnRdXSwgTGlzdFtMaXN0W0RpY3Rbc3RyLCBpbnRdXV1dOgogICAgICAgICIiIgogICAgICAgIEluZmVyIHRoZSBhdWRpbyB0aHJvdWdoIHRoZSBWQUQgbW9kZWwgYW5kIHJldHVybiB0aGUgc3BlZWNoIHRpbWVzdGFtcHMuCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiBUaGUgYXVkaW8gZmlsZSB0byBpbmZlci4KCiAgICAgICAgOnJldHVybnM6IFRoZSBzcGVlY2ggdGltZXN0YW1wcyBpbiB0aGUgYXVkaW8uIEEgbGlzdCBvZiB0aW1lc3RhbXBzIHdoZXJlIGVhY2ggdGltZXN0YW1wIGlzIGEgZGljdGlvbmFyeSB3aXRoIHRoZQogICAgICAgICAgICAgICAgIGZvbGxvd2luZyBrZXlzOgoKICAgICAgICAgICAgICAgICAqICJzdGFydCI6IFRoZSBzdGFydCBzYW1wbGUgaW5kZXggb2YgdGhlIHNwZWVjaCBpbiB0aGUgYXVkaW8uCiAgICAgICAgICAgICAgICAgKiAiZW5kIjogICBUaGUgZW5kIHNhbXBsZSBpbmRleCBvZiB0aGUgc3BlZWNoIGluIHRoZSBhdWRpby4KCiAgICAgICAgICAgICAgICAgSWYgYHBlcl9jaGFubmVsYCBpcyBUcnVlLCBhIGxpc3Qgb2YgdGltZXN0YW1wcyBwZXIgY2hhbm5lbCB3aWxsIGJlIHJldHVybmVkLgogICAgICAgICIiIgogICAgICAgICMgQ2FzdCB0byBhIG51bXB5IGFycmF5OgogICAgICAgIGF1ZGlvID0gc2VsZi5fcmVhZF9hdWRpbyhhdWRpb19maWxlKQoKICAgICAgICAjIERldGVjdCBzcGVlY2g6CiAgICAgICAgaWYgbm90IHNlbGYuX3Blcl9jaGFubmVsOgogICAgICAgICAgICByZXR1cm4gc2VsZi5fZ2V0X3NwZWVjaF90aW1lc3RhbXBzKAogICAgICAgICAgICAgICAgYXVkaW8sCiAgICAgICAgICAgICAgICBzZWxmLl9tb2RlbCwKICAgICAgICAgICAgICAgIHRocmVzaG9sZD1zZWxmLl90aHJlc2hvbGQsCiAgICAgICAgICAgICAgICBtaW5fc3BlZWNoX2R1cmF0aW9uX21zPXNlbGYuX21pbl9zcGVlY2hfZHVyYXRpb25fbXMsCiAgICAgICAgICAgICAgICBtYXhfc3BlZWNoX2R1cmF0aW9uX3M9c2VsZi5fbWF4X3NwZWVjaF9kdXJhdGlvbl9zLAogICAgICAgICAgICAgICAgbWluX3NpbGVuY2VfZHVyYXRpb25fbXM9c2VsZi5fbWluX3NpbGVuY2VfZHVyYXRpb25fbXMsCiAgICAgICAgICAgICAgICBzcGVlY2hfcGFkX21zPXNlbGYuX3NwZWVjaF9wYWRfbXMsCiAgICAgICAgICAgICAgICBzYW1wbGluZ19yYXRlPXNlbGYuX3NhbXBsaW5nX3JhdGUsCiAgICAgICAgICAgICAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzPXNlbGYuX3dpbmRvd19zaXplX3NhbXBsZXMsCiAgICAgICAgICAgICAgICByZXR1cm5fc2Vjb25kcz1zZWxmLl9yZXR1cm5fc2Vjb25kcywKICAgICAgICAgICAgKQoKICAgICAgICAjIFBlciBjaGFubmVsOgogICAgICAgIHNwZWVjaF90aW1lc3RhbXBzID0gW10KICAgICAgICBmb3IgY2hhbm5lbCBpbiBhdWRpbzoKICAgICAgICAgICAgc3BlZWNoX3RpbWVzdGFtcHMuYXBwZW5kKAogICAgICAgICAgICAgICAgc2VsZi5fZ2V0X3NwZWVjaF90aW1lc3RhbXBzKAogICAgICAgICAgICAgICAgICAgIGNoYW5uZWwsCiAgICAgICAgICAgICAgICAgICAgc2VsZi5fbW9kZWwsCiAgICAgICAgICAgICAgICAgICAgdGhyZXNob2xkPXNlbGYuX3RocmVzaG9sZCwKICAgICAgICAgICAgICAgICAgICBtaW5fc3BlZWNoX2R1cmF0aW9uX21zPXNlbGYuX21pbl9zcGVlY2hfZHVyYXRpb25fbXMsCiAgICAgICAgICAgICAgICAgICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zPXNlbGYuX21heF9zcGVlY2hfZHVyYXRpb25fcywKICAgICAgICAgICAgICAgICAgICBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcz1zZWxmLl9taW5fc2lsZW5jZV9kdXJhdGlvbl9tcywKICAgICAgICAgICAgICAgICAgICBzcGVlY2hfcGFkX21zPXNlbGYuX3NwZWVjaF9wYWRfbXMsCiAgICAgICAgICAgICAgICAgICAgc2FtcGxpbmdfcmF0ZT1zZWxmLl9zYW1wbGluZ19yYXRlLAogICAgICAgICAgICAgICAgICAgIHdpbmRvd19zaXplX3NhbXBsZXM9c2VsZi5fd2luZG93X3NpemVfc2FtcGxlcywKICAgICAgICAgICAgICAgICAgICByZXR1cm5fc2Vjb25kcz1zZWxmLl9yZXR1cm5fc2Vjb25kcywKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgKQoKICAgICAgICByZXR1cm4gc3BlZWNoX3RpbWVzdGFtcHMKCiAgICBkZWYgX3JlYWRfYXVkaW8oCiAgICAgICAgc2VsZiwKICAgICAgICBwYXRoOiBQYXRoLAogICAgKSAtPiB0b3JjaC5UZW5zb3I6CiAgICAgICAgIiIiCiAgICAgICAgUmVhZCB0aGUgYXVkaW8gZnJvbSB0aGUgZ2l2ZW4gcGF0aCBhbmQgcmV0dXJuIGl0IGFzIGEgdGVuc29yLgoKICAgICAgICA6cGFyYW0gcGF0aDogVGhlIHBhdGggdG8gdGhlIGF1ZGlvIGZpbGUuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgYXVkaW8gYXMgYSB0ZW5zb3IuCiAgICAgICAgIiIiCiAgICAgICAgIyBSZWFkIHRoZSBhdWRpbzoKICAgICAgICBhdWRpbywgc2FtcGxpbmdfcmF0ZSA9IHRvcmNoYXVkaW8ubG9hZChzdHIocGF0aCkpCgogICAgICAgICMgQ2hlY2sgaWYgdGhlIGF1ZGlvIGlzIHN0ZXJlbyBhbmQgaWYgc28sIGNvbnZlcnQgaXQgdG8gbW9ubyAob25seSBpZiBub3QgcGVyIGNoYW5uZWwpOgogICAgICAgIGlmIGF1ZGlvLnNpemUoMCkgPiAxIGFuZCBub3Qgc2VsZi5fcGVyX2NoYW5uZWw6CiAgICAgICAgICAgIGF1ZGlvID0gYXVkaW8ubWVhbihkaW09MCwga2VlcGRpbT1UcnVlKQoKICAgICAgICAjIFJlc2FtcGxlIHRoZSBhdWRpbyBpZiBuZWVkZWQ6CiAgICAgICAgaWYgc2FtcGxpbmdfcmF0ZSAhPSBzZWxmLl9zYW1wbGluZ19yYXRlOgogICAgICAgICAgICB0cmFuc2Zvcm0gPSB0b3JjaGF1ZGlvLnRyYW5zZm9ybXMuUmVzYW1wbGUoCiAgICAgICAgICAgICAgICBvcmlnX2ZyZXE9c2FtcGxpbmdfcmF0ZSwgbmV3X2ZyZXE9c2VsZi5fc2FtcGxpbmdfcmF0ZQogICAgICAgICAgICApCiAgICAgICAgICAgIGF1ZGlvID0gdHJhbnNmb3JtKGF1ZGlvKQoKICAgICAgICAjIFJldHVybiB0aGUgYXVkaW8gKHNxdWVlemUgaWYgbm90IHBlciBjaGFubmVsKToKICAgICAgICByZXR1cm4gYXVkaW8gaWYgc2VsZi5fcGVyX2NoYW5uZWwgZWxzZSBhdWRpby5zcXVlZXplKDApCgoKIzogVGhlIHZhbHVlIHRvIHNlbmQgaW50byBtdWx0aXByb2Nlc3NpbmcgcXVldWVzIHRvIHN0b3AgdGhlIHByb2Nlc3M6Cl9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLID0gIlNUT1AiCgoKZGVmIF9tdWx0aXByb2Nlc3NpbmdfY29tcGxldGVfdGFza3MoCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsIHRhc2tzX3F1ZXVlOiBRdWV1ZSwgcmVzdWx0c19xdWV1ZTogUXVldWUKKToKICAgICIiIgogICAgQ29tcGxldGUgdGhlIHRhc2tzIGluIHRoZSBnaXZlbiBxdWV1ZSBhbmQgcHV0IHRoZSByZXN1bHRzIGluIHRoZSBnaXZlbiByZXN1bHRzIHF1ZXVlLiBUaGUgZnVuY3Rpb24gd2lsbCBzdG9wIHdoZW4KICAgIHRoZSBnaXZlbiB0YXNrcyBxdWV1ZSB3aWxsIHJlY2VpdmUgdGhlIHN0b3AgbWFyay4gSXQgaXMgYWltZWQgdG8gYmUgdXNlZCB3aXRoIG11bHRpcHJvY2Vzc2luZyBhcyBhIHByb2Nlc3MuCgogICAgOnBhcmFtIHZhZF9pbml0X2t3YXJnczogVGhlIFZBRCBpbml0aWFsaXphdGlvbiBrd2FyZ3MuCiAgICA6cGFyYW0gdGFza3NfcXVldWU6ICAgICBBIHF1ZXVlIHRvIGdldCB0aGUgdGFza3MgZnJvbS4KICAgIDpwYXJhbSByZXN1bHRzX3F1ZXVlOiAgIEEgcXVldWUgdG8gcHV0IHRoZSByZXN1bHRzIGluLgogICAgIiIiCiAgICAjIEluaXRpYWxpemUgYW5kIGxvYWQgdGhlIFZBRDoKICAgIHZhZCA9IFZvaWNlQWN0aXZpdHlEZXRlY3RvcigqKnZhZF9pbml0X2t3YXJncykKICAgIHZhZC5sb2FkKCkKCiAgICAjIFN0YXJ0IGxpc3RlbmluZyB0byB0aGUgdGFza3MgcXVldWU6CiAgICB3aGlsZSBUcnVlOgogICAgICAgICMgR2V0IHRoZSB0YXNrOgogICAgICAgIHRhc2s6IEJhc2VUYXNrID0gdGFza3NfcXVldWUuZ2V0KCkKICAgICAgICBpZiB0YXNrID09IF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLOgogICAgICAgICAgICBicmVhawogICAgICAgIHRyeToKICAgICAgICAgICAgIyBSdW4gdGhlIGZpbGUgdGhyb3VnaCB0aGUgVkFEOgogICAgICAgICAgICBzcGVlY2hfdGltZXN0YW1wcyA9IHZhZC5kZXRlY3Rfdm9pY2UoYXVkaW9fZmlsZT10YXNrLmF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSByZXN1bHQ6CiAgICAgICAgICAgIHJlc3VsdHNfcXVldWUucHV0KChGYWxzZSwgdGFzay5nZXRfcmVzdWx0KCkpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIENvbGxlY3QgdGhlIGVycm9yOgogICAgICAgICAgICByZXN1bHRzX3F1ZXVlLnB1dCgoVHJ1ZSwgKHRhc2suYXVkaW9fZmlsZS5uYW1lLCBzdHIoZXhjZXB0aW9uKSkpKQoKICAgICMgTWFyayB0aGUgZW5kIG9mIHRoZSB0YXNrczoKICAgIHJlc3VsdHNfcXVldWUucHV0KF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLKQoKCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgpfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBkZXRlY3Rfdm9pY2UoCiAgICAjIElucHV0IGt3YXJnczoKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBQYXRoLCBMaXN0W1VuaW9uW3N0ciwgUGF0aF1dXSwKICAgICMgTW9kZWwgbG9hZGluZyBrd2FyZ3M6CiAgICB1c2Vfb25ueDogYm9vbCA9IFRydWUsCiAgICBmb3JjZV9vbm54X2NwdTogYm9vbCA9IFRydWUsCiAgICAjIERldGVjdGlvbiBrd2FyZ3M6CiAgICB0aHJlc2hvbGQ6IGZsb2F0ID0gMC41LAogICAgc2FtcGxpbmdfcmF0ZTogaW50ID0gMTZfMDAwLAogICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tczogaW50ID0gMjUwLAogICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiBmbG9hdCA9IGZsb2F0KCJpbmYiKSwKICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBpbnQgPSAxMDAsCiAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzOiBpbnQgPSA1MTIsCiAgICBzcGVlY2hfcGFkX21zOiBpbnQgPSAzMCwKICAgIHJldHVybl9zZWNvbmRzOiBib29sID0gRmFsc2UsCiAgICBwZXJfY2hhbm5lbDogYm9vbCA9IEZhbHNlLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHZvaWNlIGFjdGl2aXR5IGRldGVjdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtCiAgICBodHRwczovL2dpdGh1Yi5jb20vc25ha2VyczQvc2lsZXJvLXZhZC4gVGhlIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIKICAgIFZBRCB0aW1lc3RhbXBzIGRpY3Rpb25hcmllcyBhcyB2YWx1ZS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICB7InN0YXJ0IjogMCwgImVuZCI6IDE2MDAwfSwKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAxNjAwMCwgImVuZCI6IDMyMDAwfSwKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAzMjAwMCwgImVuZCI6IDQ4MDAwfSwKICAgICAgICAgICAgICAgIC4uLgogICAgICAgICAgICBdLAogICAgICAgICAgICAiZmlsZV8yLndhdiI6IFsKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAwLCAiZW5kIjogMTYwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDE2MDAwLCAiZW5kIjogMzIwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDMyMDAwLCAiZW5kIjogNDgwMDB9LAogICAgICAgICAgICAgICAgLi4uCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgIC4uLgogICAgICAgIH0KCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgICAgICBUaGUgcGF0aCB0byB0aGUgYXVkaW8gZmlsZXMgdG8gZGlhcml6ZS4gQ2FuIGJlIGEgcGF0aCB0byBhIHNpbmdsZSBmaWxlLCBhIHBhdGggdG8gYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkaXJlY3Rvcnkgb3IgYSBsaXN0IG9mIHBhdGhzIHRvIGZpbGVzLgogICAgOnBhcmFtIHVzZV9vbm54OiAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHVzZSBPTk5YIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgIDpwYXJhbSBmb3JjZV9vbm54X2NwdTogICAgICAgICAgV2hldGhlciB0byBmb3JjZSBPTk5YIHRvIHVzZSBDUFUgZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIHRocmVzaG9sZDogICAgICAgICAgICAgICBTcGVlY2ggdGhyZXNob2xkLiBTaWxlcm8gVkFEIG91dHB1dHMgc3BlZWNoIHByb2JhYmlsaXRpZXMgZm9yIGVhY2ggYXVkaW8gY2h1bmssCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHByb2JhYmlsaXRpZXMgQUJPVkUgdGhpcyB2YWx1ZSBhcmUgY29uc2lkZXJlZCBhcyBTUEVFQ0guIEl0IGlzIGJldHRlciB0byB0dW5lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoaXMgcGFyYW1ldGVyIGZvciBlYWNoIGRhdGFzZXQgc2VwYXJhdGVseSwgYnV0ICJsYXp5IiAwLjUgaXMgcHJldHR5IGdvb2QgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vc3QgZGF0YXNldHMuCiAgICA6cGFyYW0gc2FtcGxpbmdfcmF0ZTogICAgICAgICAgIEN1cnJlbnRseSwgc2lsZXJvIFZBRCBtb2RlbHMgc3VwcG9ydCA4MDAwIGFuZCAxNjAwMCBzYW1wbGUgcmF0ZXMuCiAgICA6cGFyYW0gbWluX3NwZWVjaF9kdXJhdGlvbl9tczogIEZpbmFsIHNwZWVjaCBjaHVua3Mgc2hvcnRlciBtaW5fc3BlZWNoX2R1cmF0aW9uX21zIGFyZSB0aHJvd24gb3V0LgogICAgOnBhcmFtIG1heF9zcGVlY2hfZHVyYXRpb25fczogICBNYXhpbXVtIGR1cmF0aW9uIG9mIHNwZWVjaCBjaHVua3MgaW4gc2Vjb25kcy4gQ2h1bmtzIGxvbmdlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtYXhfc3BlZWNoX2R1cmF0aW9uX3NgIHdpbGwgYmUgc3BsaXQgYXQgdGhlIHRpbWVzdGFtcCBvZiB0aGUgbGFzdCBzaWxlbmNlIHRoYXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFzdHMgbW9yZSB0aGFuIDEwMG1zIChpZiBhbnkpLCB0byBwcmV2ZW50IGFnZ3Jlc3NpdmUgY3V0dGluZy4gT3RoZXJ3aXNlLCB0aGV5IHdpbGwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYmUgc3BsaXQgYWdncmVzc2l2ZWx5IGp1c3QgYmVmb3JlIG1heF9zcGVlY2hfZHVyYXRpb25fcy4KICAgIDpwYXJhbSBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogSW4gdGhlIGVuZCBvZiBlYWNoIHNwZWVjaCBjaHVuayB3YWl0IGZvciBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyBiZWZvcmUgc2VwYXJhdGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpdC4KICAgIDpwYXJhbSB3aW5kb3dfc2l6ZV9zYW1wbGVzOiAgICAgQXVkaW8gY2h1bmtzIG9mIHdpbmRvd19zaXplX3NhbXBsZXMgc2l6ZSBhcmUgZmVkIHRvIHRoZSBzaWxlcm8gVkFEIG1vZGVsLgoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV0FSTklORyEgU2lsZXJvIFZBRCBtb2RlbHMgd2VyZSB0cmFpbmVkIHVzaW5nIDUxMiwgMTAyNCwgMTUzNiBzYW1wbGVzIGZvciAxNjAwMAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgcmF0ZSBhbmQgMjU2LCA1MTIsIDc2OCBzYW1wbGVzIGZvciA4MDAwIHNhbXBsZSByYXRlLiBWYWx1ZXMgb3RoZXIgdGhhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGVzZSBtYXkgYWZmZWN0IG1vZGVsIHBlcmZvcm1hbmNlIQogICAgOnBhcmFtIHNwZWVjaF9wYWRfbXM6ICAgICAgICAgICBGaW5hbCBzcGVlY2ggY2h1bmtzIGFyZSBwYWRkZWQgYnkgc3BlZWNoX3BhZF9tcyBlYWNoIHNpZGUuCiAgICA6cGFyYW0gcmV0dXJuX3NlY29uZHM6ICAgICAgICAgIFdoZXRoZXIgcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2Vjb25kcy4gRmFsc2UgbWVhbnMgdG8gcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2FtcGxlcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoZGVmYXVsdCAtIEZhbHNlKS4KICAgIDpwYXJhbSBwZXJfY2hhbm5lbDogICAgICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGltZXN0YW1wcyBwZXIgY2hhbm5lbCAoZGVmYXVsdCAtIEZhbHNlKS4gVGhpcyB3aWxsIHJ1biBWQUQgb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZWFjaCBjaGFubmVsIHNlcGFyYXRlbHkgYW5kIHJldHVybiBhIGxpc3Qgb2YgdGltZXN0YW1wcyBwZXIgY2hhbm5lbC4KICAgIDpwYXJhbSB1c2VfbXVsdGlwcm9jZXNzaW5nOiAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZSBmb3IgbXVsdGlwcm9jZXNzaW5nLiBJZiAwLCBubyBtdWx0aXByb2Nlc3Npbmcgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkLiBEZWZhdWx0IGlzIDAuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgICAgIFZlcmJvc2l0eS4KICAgICIiIgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIEdldCB0aGUgaW5wdXQgYXVkaW8gZmlsZXMgdG8gdHJhbnNjcmliZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJDb2xsZWN0aW5nIGF1ZGlvIGZpbGVzLiIpCiAgICBhdWRpb19maWxlcyA9IF9nZXRfYXVkaW9fZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiQ29sbGVjdGVkIHtsZW4oYXVkaW9fZmlsZXMpfSBhdWRpbyBmaWxlcy4iKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgdHJhbnNjcmlwdGlvbiBwaXBlbGluZToKICAgIHZhZF9pbml0X2t3YXJncyA9IHsKICAgICAgICAidXNlX29ubngiOiB1c2Vfb25ueCwKICAgICAgICAiZm9yY2Vfb25ueF9jcHUiOiBmb3JjZV9vbm54X2NwdSwKICAgICAgICAidGhyZXNob2xkIjogdGhyZXNob2xkLAogICAgICAgICJzYW1wbGluZ19yYXRlIjogc2FtcGxpbmdfcmF0ZSwKICAgICAgICAibWluX3NwZWVjaF9kdXJhdGlvbl9tcyI6IG1pbl9zcGVlY2hfZHVyYXRpb25fbXMsCiAgICAgICAgIm1heF9zcGVlY2hfZHVyYXRpb25fcyI6IG1heF9zcGVlY2hfZHVyYXRpb25fcywKICAgICAgICAibWluX3NpbGVuY2VfZHVyYXRpb25fbXMiOiBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcywKICAgICAgICAid2luZG93X3NpemVfc2FtcGxlcyI6IHdpbmRvd19zaXplX3NhbXBsZXMsCiAgICAgICAgInNwZWVjaF9wYWRfbXMiOiBzcGVlY2hfcGFkX21zLAogICAgICAgICJyZXR1cm5fc2Vjb25kcyI6IHJldHVybl9zZWNvbmRzLAogICAgICAgICJwZXJfY2hhbm5lbCI6IHBlcl9jaGFubmVsLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcih0YXNrX3R5cGU9QmFzZVRhc2spCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEZXRlY3Rpbmcgdm9pY2UiLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEZXRlY3Rpbmcgdm9pY2UiLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBkaWFyaXplKAogICAgIyBJbnB1dCAvIE91dHB1dCBrd2FyZ3M6CiAgICBkYXRhX3BhdGg6IFVuaW9uW3N0ciwgUGF0aCwgTGlzdFtVbmlvbltzdHIsIFBhdGhdXV0sCiAgICAjIE1vZGVsIGxvYWRpbmcga3dhcmdzOgogICAgdXNlX29ubng6IGJvb2wgPSBUcnVlLAogICAgZm9yY2Vfb25ueF9jcHU6IGJvb2wgPSBUcnVlLAogICAgIyBEZXRlY3Rpb24ga3dhcmdzOgogICAgdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgIHNhbXBsaW5nX3JhdGU6IGludCA9IDE2XzAwMCwKICAgIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6IGludCA9IDI1MCwKICAgIG1heF9zcGVlY2hfZHVyYXRpb25fczogZmxvYXQgPSBmbG9hdCgiaW5mIiksCiAgICBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogaW50ID0gMTAwLAogICAgd2luZG93X3NpemVfc2FtcGxlczogaW50ID0gNTEyLAogICAgc3BlZWNoX3BhZF9tczogaW50ID0gMzAsCiAgICAjIERpYXJpemF0aW9uIGt3YXJnczoKICAgIHNwZWFrZXJfbGFiZWxzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHNwZWVjaCBkaWFyaXphdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtIGh0dHBzOi8vZ2l0aHViLmNvbS9zbmFrZXJzNC9zaWxlcm8tdmFkLgogICAgVGhlIHNwZWVjaCBkaWFyaXphdGlvbiBpcyBwZXJmb3JtZWQgcGVyIGNoYW5uZWwgc28gdGhhdCBlYWNoIGNoYW5uZWwgaW4gdGhlIGF1ZGlvIGJlbG9uZyB0byBhIGRpZmZlcmVudCBzcGVha2VyLiBUaGUKICAgIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIgZGlhcml6YXRpb24gYXMgdmFsdWUuIEEgZGlhcml6YXRpb24gaXMgYSBsaXN0CiAgICBvZiB0dXBsZXM6IChzdGFydCwgZW5kLCBzcGVha2VyX2xhYmVsKS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgImZpbGVfMi53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgLi4uCiAgICAgICAgfQoKCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICAgICAgIFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlcyB0byBkaWFyaXplLiBDYW4gYmUgYSBwYXRoIHRvIGEgc2luZ2xlIGZpbGUsIGEgcGF0aCB0byBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRpcmVjdG9yeSBvciBhIGxpc3Qgb2YgcGF0aHMgdG8gZmlsZXMuCiAgICA6cGFyYW0gdXNlX29ubng6ICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIE9OTlggZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIGZvcmNlX29ubnhfY3B1OiAgICAgICAgICBXaGV0aGVyIHRvIGZvcmNlIE9OTlggdG8gdXNlIENQVSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIFRydWUuCiAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHJvYmFiaWxpdGllcyBBQk9WRSB0aGlzIHZhbHVlIGFyZSBjb25zaWRlcmVkIGFzIFNQRUVDSC4gSXQgaXMgYmV0dGVyIHRvIHR1bmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhpcyBwYXJhbWV0ZXIgZm9yIGVhY2ggZGF0YXNldCBzZXBhcmF0ZWx5LCBidXQgImxhenkiIDAuNSBpcyBwcmV0dHkgZ29vZCBmb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgIDpwYXJhbSBzYW1wbGluZ19yYXRlOiAgICAgICAgICAgQ3VycmVudGx5LCBzaWxlcm8gVkFEIG1vZGVscyBzdXBwb3J0IDgwMDAgYW5kIDE2MDAwIHNhbXBsZSByYXRlcy4KICAgIDpwYXJhbSBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiAgRmluYWwgc3BlZWNoIGNodW5rcyBzaG9ydGVyIG1pbl9zcGVlY2hfZHVyYXRpb25fbXMgYXJlIHRocm93biBvdXQuCiAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYG1heF9zcGVlY2hfZHVyYXRpb25fc2Agd2lsbCBiZSBzcGxpdCBhdCB0aGUgdGltZXN0YW1wIG9mIHRoZSBsYXN0IHNpbGVuY2UgdGhhdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXN0cyBtb3JlIHRoYW4gMTAwbXMgKGlmIGFueSksIHRvIHByZXZlbnQgYWdncmVzc2l2ZSBjdXR0aW5nLiBPdGhlcndpc2UsIHRoZXkgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgOnBhcmFtIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBJbiB0aGUgZW5kIG9mIGVhY2ggc3BlZWNoIGNodW5rIHdhaXQgZm9yIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zIGJlZm9yZSBzZXBhcmF0aW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGl0LgogICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSByYXRlIGFuZCAyNTYsIDUxMiwgNzY4IHNhbXBsZXMgZm9yIDgwMDAgc2FtcGxlIHJhdGUuIFZhbHVlcyBvdGhlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXNlIG1heSBhZmZlY3QgbW9kZWwgcGVyZm9ybWFuY2UhCiAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgIDpwYXJhbSBzcGVha2VyX2xhYmVsczogICAgICAgICAgVGhlIHNwZWFrZXIgbGFiZWxzIHRvIHVzZSBmb3IgdGhlIGRpYXJpemF0aW9uLiBJZiBub3QgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5hbWVkICJzcGVha2VyXzAiLCAic3BlYWtlcl8xIiwgZXRjLgogICAgOnBhcmFtIHVzZV9tdWx0aXByb2Nlc3Npbmc6ICAgICBUaGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8gdXNlIGZvciBtdWx0aXByb2Nlc3NpbmcuIElmIDAsIG5vIG11bHRpcHJvY2Vzc2luZyB3aWxsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHVzZWQuIERlZmF1bHQgaXMgMC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgVmVyYm9zaXR5LgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgYXVkaW8gZmlsZXMuIikKICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbihhdWRpb19maWxlcyl9IGF1ZGlvIGZpbGVzLiIpCgogICAgIyBJbml0aWFsaXplIHRoZSB0cmFuc2NyaXB0aW9uIHBpcGVsaW5lOgogICAgdmFkX2luaXRfa3dhcmdzID0gewogICAgICAgICJ1c2Vfb25ueCI6IHVzZV9vbm54LAogICAgICAgICJmb3JjZV9vbm54X2NwdSI6IGZvcmNlX29ubnhfY3B1LAogICAgICAgICJ0aHJlc2hvbGQiOiB0aHJlc2hvbGQsCiAgICAgICAgInNhbXBsaW5nX3JhdGUiOiBzYW1wbGluZ19yYXRlLAogICAgICAgICJtaW5fc3BlZWNoX2R1cmF0aW9uX21zIjogbWluX3NwZWVjaF9kdXJhdGlvbl9tcywKICAgICAgICAibWF4X3NwZWVjaF9kdXJhdGlvbl9zIjogbWF4X3NwZWVjaF9kdXJhdGlvbl9zLAogICAgICAgICJtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyI6IG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICJ3aW5kb3dfc2l6ZV9zYW1wbGVzIjogd2luZG93X3NpemVfc2FtcGxlcywKICAgICAgICAic3BlZWNoX3BhZF9tcyI6IHNwZWVjaF9wYWRfbXMsCiAgICAgICAgInJldHVybl9zZWNvbmRzIjogVHJ1ZSwKICAgICAgICAicGVyX2NoYW5uZWwiOiBUcnVlLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcigKICAgICAgICB0YXNrX3R5cGU9U3BlZWNoRGlhcml6YXRpb25UYXNrLCB0YXNrX2t3YXJncz17InNwZWFrZXJfbGFiZWxzIjogc3BlYWtlcl9sYWJlbHN9CiAgICApCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgZGF0YV9wYXRoOiBVbmlvbltQYXRoLCBzdHIsIGxpc3RdLAopIC0+IExpc3RbUGF0aF06CiAgICAiIiIKICAgIEdldCB0aGUgYXVkaW8gZmlsZXMgZnJvbSB0aGUgZGF0YSBwYXRoLiBJZiBhIHBhdGggdG8gYSBkaXJlY3RvcnkgaXMgZ2l2ZW4sIGFsbCBmaWxlcyBpbiB0aGUgZGlyZWN0b3J5IHdpbGwgYmUKICAgIGNvbGxlY3RlZC4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiBUaGUgZGF0YSBwYXRoIHRvIGNvbGxlY3QgdGhlIGF1ZGlvIGZpbGVzIGZyb20uCgogICAgOnJldHVybnM6IFRoZSBhdWRpbyBmaWxlcyBsaXN0LgogICAgIiIiCiAgICAjIENoZWNrIGlmIGdpdmVuIGEgbGlzdCBvZiBwYXRoczoKICAgIGlmIGlzaW5zdGFuY2UoZGF0YV9wYXRoLCBsaXN0KToKICAgICAgICBhdWRpb19maWxlcyA9IFtdCiAgICAgICAgZm9yIHBhdGggaW4gZGF0YV9wYXRoOgogICAgICAgICAgICBhdWRpb19maWxlcy5leHRlbmQoX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9cGF0aCkpCiAgICAgICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgogICAgIyBDaGVjayBpZiBnaXZlbiBhIHNpbmdsZSBzdHJpbmcgcGF0aCB0byBjYXN0IGl0IHRvIGEgYHBhdGhsaWIuUGF0aGA6CiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgc3RyKToKICAgICAgICBkYXRhX3BhdGggPSBQYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQoKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICBhdWRpb19maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gW2RhdGFfcGF0aF0KICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJVbnJlY29nbml6ZWQgZGF0YSBwYXRoLiBUaGUgcGFyYW1ldGVyIGBkYXRhX3BhdGhgIG11c3QgYmUgYSB2YWxpZCBwYXRoIHRvIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgIgogICAgICAgICAgICBmImZpbGUuIEdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgoKZGVmIF9ydW4oCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgTG9hZCBhIFZBRCBhbmQgdXNlIGl0IHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcgdGhlIGdpdmVuIHRhc2sgY3JlYXRvci4KCiAgICA6cGFyYW0gYXVkaW9fZmlsZXM6ICAgICBUaGUgYXVkaW8gZmlsZXMgdG8gdXNlLgogICAgOnBhcmFtIGRlc2NyaXB0aW9uOiAgICAgVGhlIGRlc2NyaXB0aW9uIHRvIHVzZSBmb3IgdGhlIHByb2dyZXNzIGJhci4KICAgIDpwYXJhbSB2YWRfaW5pdF9rd2FyZ3M6IFRoZSBWQUQgaW5pdGlhbGl6YXRpb24ga2V5d29yZCBhcmd1bWVudHMuCiAgICA6cGFyYW0gdGFza19jcmVhdG9yOiAgICBUaGUgdGFzayBjcmVhdG9yIHRvIHVzZSB0byBjcmVhdGUgdGhlIHRhc2tzLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgVmVyYm9zaXR5LgoKICAgIDpyZXR1cm5zOiBUaGUgY29sbGVjdGVkIHJlc3VsdHMuCiAgICAiIiIKICAgICMgTG9hZCB0aGUgVkFEOgogICAgdmFkID0gVm9pY2VBY3Rpdml0eURldGVjdG9yKCoqdmFkX2luaXRfa3dhcmdzKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJMb2FkaW5nIHRoZSBWQUQgbW9kZWwuIikKICAgIHZhZC5sb2FkKCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJWQUQgbW9kZWwgbG9hZGVkLiIpCgogICAgIyBSdW4gdGhlIFZBRCBvbiB0aGUgYXVkaW8gZmlsZXMgYW5kIGNvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsCiAgICAgICAgZGVzYz1kZXNjcmlwdGlvbiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICAgICB0b3RhbD1sZW4oYXVkaW9fZmlsZXMpLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2sgPSB0YXNrX2NyZWF0b3IuY3JlYXRlX3Rhc2soYXVkaW9fZmlsZT1hdWRpb19maWxlKQogICAgICAgICAgICAjIFJ1biB0aGUgZmlsZSB0aHJvdWdoIHRoZSBWQUQ6CiAgICAgICAgICAgIHNwZWVjaF90aW1lc3RhbXBzID0gdmFkLmRldGVjdF92b2ljZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSByZXN1bHQ6CiAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKChGYWxzZSwgdGFzay5nZXRfcmVzdWx0KCkpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIENvbGxlY3QgdGhlIGVycm9yOgogICAgICAgICAgICByZXN1bHRzLmFwcGVuZCgoVHJ1ZSwgKGF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpKSkKCiAgICByZXR1cm4gcmVzdWx0cwoKCmRlZiBfcGFyYWxsZWxfcnVuKAogICAgbl93b3JrZXJzOiBpbnQsCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgUnVuIG11bHRpcGxlIFZBRCB3b3JrZXJzIHdpdGggbXVsdGlwcm9jZXNzaW5nIHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcKICAgIHRoZSBnaXZlbiB0YXNrIGNyZWF0b3IuCgogICAgOnBhcmFtIG5fd29ya2VyczogICAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZS4KICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgIFRoZSBhdWRpbyBmaWxlcyB0byB1c2UuCiAgICA6cGFyYW0gZGVzY3JpcHRpb246ICAgICBUaGUgZGVzY3JpcHRpb24gdG8gdXNlIGZvciB0aGUgcHJvZ3Jlc3MgYmFyLgogICAgOnBhcmFtIHZhZF9pbml0X2t3YXJnczogVGhlIFZBRCBpbml0aWFsaXphdGlvbiBrZXl3b3JkIGFyZ3VtZW50cy4KICAgIDpwYXJhbSB0YXNrX2NyZWF0b3I6ICAgIFRoZSB0YXNrIGNyZWF0b3IgdG8gdXNlIHRvIGNyZWF0ZSB0aGUgdGFza3MuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICBWZXJib3NpdHkuCgogICAgOnJldHVybnM6IFRoZSBjb2xsZWN0ZWQgcmVzdWx0cy4KICAgICIiIgogICAgIyBDaGVjayB0aGUgbnVtYmVyIG9mIHdvcmtlcnM6CiAgICBpZiBuX3dvcmtlcnMgPiBsZW4oYXVkaW9fZmlsZXMpOgogICAgICAgIF9MT0dHRVIud2FybmluZygKICAgICAgICAgICAgZiJUaGUgbnVtYmVyIG9mIHdvcmtlcnMgKHtuX3dvcmtlcnN9KSBpcyBsYXJnZXIgdGhhbiB0aGUgbnVtYmVyIG9mIGF1ZGlvIGZpbGVzICh7bGVuKGF1ZGlvX2ZpbGVzKX0pLiAiCiAgICAgICAgICAgIGYiU2V0dGluZyB0aGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8ge2xlbihhdWRpb19maWxlcyl9LiIKICAgICAgICApCiAgICAgICAgbl93b3JrZXJzID0gbGVuKGF1ZGlvX2ZpbGVzKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgbXVsdGlwcm9jZXNzaW5nIHF1ZXVlczoKICAgIHRhc2tzX3F1ZXVlID0gUXVldWUobWF4c2l6ZT1uX3dvcmtlcnMgKiAyKQogICAgcmVzdWx0c19xdWV1ZSA9IFF1ZXVlKCkKCiAgICAjIEluaXRpYWxpemUgdGhlIG11bHRpcHJvY2Vzc2luZyBwcm9jZXNzZXM6CiAgICB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzID0gWwogICAgICAgIFByb2Nlc3MoCiAgICAgICAgICAgIHRhcmdldD1fbXVsdGlwcm9jZXNzaW5nX2NvbXBsZXRlX3Rhc2tzLAogICAgICAgICAgICBrd2FyZ3M9ewogICAgICAgICAgICAgICAgInZhZF9pbml0X2t3YXJncyI6IHZhZF9pbml0X2t3YXJncywKICAgICAgICAgICAgICAgICJ0YXNrc19xdWV1ZSI6IHRhc2tzX3F1ZXVlLAogICAgICAgICAgICAgICAgInJlc3VsdHNfcXVldWUiOiByZXN1bHRzX3F1ZXVlLAogICAgICAgICAgICB9LAogICAgICAgICkKICAgICAgICBmb3IgXyBpbiByYW5nZShuX3dvcmtlcnMpCiAgICBdCgogICAgIyBTdGFydCB0aGUgbXVsdGlwcm9jZXNzaW5nIHByb2Nlc3NlczoKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTG9hZGluZyB0aGUgVkFEIG1vZGVsIChwZXIgcHJvY2VzcykuIikKICAgIGZvciBwIGluIHRhc2tfY29tcGxldGlvbl9wcm9jZXNzZXM6CiAgICAgICAgcC5zdGFydCgpCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiVkFEIG1vZGVsIGxvYWRlZC4iKQoKICAgICMgUHV0IHRoZSB0YXNrcyBpbiB0aGUgcXVldWUgKHRoZSBwcm9ncmVzcyBiYXIgaXMgbm90IGFjY3VyYXRlIGFzIGl0IGlzIHVwZGF0aW5nIGJ5IHRoZSBxdWV1ZSB3aGljaCBoYXMgYSBtYXggc2l6ZQogICAgIyBvZiAyKm5fd29ya2VycyBzbyB0aGUgcHJvZ3Jlc3MgYmFyIHdvbid0IGJlIHRvbyBvZmYgLSBiZXR0ZXIgdGhhbiBub3RoaW5nKToKICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsCiAgICAgICAgZGVzYz1kZXNjcmlwdGlvbiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICAgICB0b3RhbD1sZW4oYXVkaW9fZmlsZXMpLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApOgogICAgICAgICMgUHV0IHRoZSB0YXNrIGluIHRoZSBxdWV1ZToKICAgICAgICB0YXNrc19xdWV1ZS5wdXQodGFza19jcmVhdG9yLmNyZWF0ZV90YXNrKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSkpCgogICAgIyBQdXQgdGhlIHN0b3AgbWFya3MgaW4gdGhlIHF1ZXVlOgogICAgZm9yIF8gaW4gcmFuZ2Uobl93b3JrZXJzKToKICAgICAgICB0YXNrc19xdWV1ZS5wdXQoX01VTFRJUFJPQ0VTU0lOR19TVE9QX01BUkspCgogICAgIyBXYWl0IGZvciB0aGUgcHJvY2Vzc2VzIHRvIGZpbmlzaDoKICAgIGZvciBwIGluIHRhc2tfY29tcGxldGlvbl9wcm9jZXNzZXM6CiAgICAgICAgcC5qb2luKCkKCiAgICAjIENvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIHN0b3BfbWFya3NfY291bnRlciA9IDAKICAgIHdoaWxlIFRydWU6CiAgICAgICAgIyBHZXQgYSByZXN1bHQgZnJvbSB0aGUgcXVldWU6CiAgICAgICAgcmVzdWx0OiBUdXBsZVtib29sLCBUdXBsZVtzdHIsIGxpc3RdXSA9IHJlc3VsdHNfcXVldWUuZ2V0KCkKICAgICAgICBpZiByZXN1bHQgPT0gX01VTFRJUFJPQ0VTU0lOR19TVE9QX01BUks6CiAgICAgICAgICAgIHN0b3BfbWFya3NfY291bnRlciArPSAxCiAgICAgICAgICAgIGlmIHN0b3BfbWFya3NfY291bnRlciA9PSBuX3dvcmtlcnM6CiAgICAgICAgICAgICAgICBicmVhawogICAgICAgIGVsc2U6CiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgcmVzdWx0OgogICAgICAgICAgICByZXN1bHRzLmFwcGVuZChyZXN1bHQpCgogICAgcmV0dXJuIHJlc3VsdHMKCgpkZWYgX3Byb2Nlc3NfcmVzdWx0cygKICAgIHJlc3VsdHM6IExpc3RbVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBsaXN0XV1dLCB2ZXJib3NlOiBib29sCikgLT4gVHVwbGVbZGljdCwgZGljdF06CiAgICAiIiIKICAgIFByb2Nlc3MgdGhlIHJlc3VsdHMgb2YgdGhlIHRhc2tzLgoKICAgIDpwYXJhbSByZXN1bHRzOiBUaGUgcmVzdWx0cyB0byBwcm9jZXNzLgogICAgOnBhcmFtIHZlcmJvc2U6IFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIHByb2Nlc3NlZCByZXN1bHRzIGFzIGEgdHVwbGUgb2Ygc3VjY2Vzc2VzIGFuZCBlcnJvcnMuCiAgICAiIiIKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJTdW1tYXJpemluZyB0aGUgcmVzdWx0cy4iKQogICAgc3VjY2Vzc2VzID0ge30KICAgIGVycm9ycyA9IHt9CiAgICBmb3IgaXNfZXJyb3IsIHJlc3VsdCBpbiByZXN1bHRzOgogICAgICAgIGlmIGlzX2Vycm9yOgogICAgICAgICAgICBlcnJvcnNbcmVzdWx0WzBdXSA9IHJlc3VsdFsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHN1Y2Nlc3Nlc1tyZXN1bHRbMF1dID0gcmVzdWx0WzFdCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkRvbmUgKHtsZW4oc3VjY2Vzc2VzKX0ve2xlbihzdWNjZXNzZXMpICsgbGVuKGVycm9ycyl9KVxuIikKCiAgICByZXR1cm4gc3VjY2Vzc2VzLCBlcnJvcnMK + functionSourceCode: IyBDb3B5cmlnaHQgMjAyNCBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgbG9nZ2luZwpmcm9tIG11bHRpcHJvY2Vzc2luZyBpbXBvcnQgUHJvY2VzcywgUXVldWUKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwZXMgaW1wb3J0IEZ1bmN0aW9uVHlwZQpmcm9tIHR5cGluZyBpbXBvcnQgRGljdCwgTGlzdCwgVHVwbGUsIFR5cGUsIFVuaW9uCgppbXBvcnQgdG9yY2gKaW1wb3J0IHRvcmNoYXVkaW8KZnJvbSB0cWRtIGltcG9ydCB0cWRtCgoKY2xhc3MgQmFzZVRhc2s6CiAgICAiIiIKICAgIEEgYmFzZSBjbGFzcyBmb3IgYSB0YXNrIHRvIGNvbXBsZXRlIGFmdGVyIFZBRC4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCBhdWRpb19maWxlOiBQYXRoKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSBiYXNlIHRhc2suCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiBUaGUgYXVkaW8gZmlsZSBhc3NpZ25lZCB0byB0aGUgdGFzay4KICAgICAgICAiIiIKICAgICAgICAjIFN0b3JlIHRoZSBhdWRpbyBmaWxlOgogICAgICAgIHNlbGYuX2F1ZGlvX2ZpbGUgPSBhdWRpb19maWxlCgogICAgICAgICMgUHJlcGFyZSB0aGUgcmVzdWx0OgogICAgICAgIHNlbGYuX3Jlc3VsdCA9IE5vbmUKCiAgICBAcHJvcGVydHkKICAgIGRlZiBhdWRpb19maWxlKHNlbGYpIC0+IFBhdGg6CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSBhdWRpbyBmaWxlIG9mIHRoZSB0YXNrLgoKICAgICAgICA6cmV0dXJuczogVGhlIGF1ZGlvIGZpbGUgb2YgdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2F1ZGlvX2ZpbGUKCiAgICBkZWYgZG9fdGFzaygKICAgICAgICBzZWxmLCBzcGVlY2hfdGltZXN0YW1wczogVW5pb25bTGlzdFtEaWN0W3N0ciwgaW50XV0sIExpc3RbTGlzdFtEaWN0W3N0ciwgaW50XV1dXQogICAgKToKICAgICAgICAiIiIKICAgICAgICBEbyB0aGUgdGFzayBvbiB0aGUgZ2l2ZW4gc3BlZWNoIHRpbWVzdGFtcHMuIFRoZSBiYXNlIHRhc2sgd2lsbCBzaW1wbHkgc2F2ZSB0aGUgc3BlZWNoIHRpbWVzdGFtcHMgYXMgdGhlIHJlc3VsdC4KCiAgICAgICAgOnBhcmFtIHNwZWVjaF90aW1lc3RhbXBzOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgdG8gZG8gdGhlIHRhc2sgb24gYXMgb3V0cHV0dGVkIGZyb20gdGhlIFZBRC4KICAgICAgICAiIiIKICAgICAgICBzZWxmLl9yZXN1bHQgPSBzcGVlY2hfdGltZXN0YW1wcwoKICAgIGRlZiBnZXRfcmVzdWx0KHNlbGYpIC0+IFR1cGxlW3N0ciwgbGlzdF06CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSByZXN1bHQgb2YgdGhlIHRhc2suIEEgdHVwbGUgb2YgdGhlIGF1ZGlvIGZpbGUgbmFtZSBhbmQgdGhlIHJlc3VsdC4KCiAgICAgICAgOnJldHVybnM6IFRoZSByZXN1bHQgb2YgdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2F1ZGlvX2ZpbGUubmFtZSwgc2VsZi5fcmVzdWx0CgogICAgZGVmIHRvX3R1cGxlKHNlbGYpIC0+IFR1cGxlW3N0ciwgZGljdF06CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCB0aGUgdGFzayB0byBhIHR1cGxlIHRvIHJlY29uc3RydWN0IGl0IGxhdGVyICh1c2VkIGZvciBtdWx0aXByb2Nlc3NpbmcgdG8gcGFzcyBpbiBxdWV1ZSkuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY29udmVydGVkIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX19jbGFzc19fLl9fbmFtZV9fLCB7ImF1ZGlvX2ZpbGUiOiBzZWxmLl9hdWRpb19maWxlfQoKCmNsYXNzIFNwZWVjaERpYXJpemF0aW9uVGFzayhCYXNlVGFzayk6CiAgICAiIiIKICAgIEEgc3BlZWNoIGRpYXJpemF0aW9uIHRhc2suIFRoZSB0YXNrIHdpbGwgZGlhcml6ZSB0aGUgVkFEIHNwZWVjaCB0aW1lc3RhbXBzIGludG8gc3BlYWtlcnMuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgYXVkaW9fZmlsZTogUGF0aCwgc3BlYWtlcl9sYWJlbHM6IExpc3Rbc3RyXSk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSB0aGUgc3BlZWNoIGRpYXJpemF0aW9uIHRhc2suCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiAgICAgVGhlIGF1ZGlvIGZpbGUgYXNzaWduZWQgdG8gdGhlIHRhc2suCiAgICAgICAgOnBhcmFtIHNwZWFrZXJfbGFiZWxzOiBUaGUgc3BlYWtlciBsYWJlbHMgdG8gdXNlIGZvciB0aGUgZGlhcml6YXRpb24uIElmIG5vdCBnaXZlbiwgdGhlIHNwZWFrZXJzIHdpbGwgYmUgbmFtZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzcGVha2VyXzAiLCAic3BlYWtlcl8xIiwgZXRjLgogICAgICAgICIiIgogICAgICAgIHN1cGVyKCkuX19pbml0X18oYXVkaW9fZmlsZT1hdWRpb19maWxlKQogICAgICAgIHNlbGYuX3NwZWFrZXJfbGFiZWxzID0gc3BlYWtlcl9sYWJlbHMKCiAgICBkZWYgZG9fdGFzayhzZWxmLCBzcGVlY2hfdGltZXN0YW1wczogTGlzdFtMaXN0W0RpY3Rbc3RyLCBpbnRdXV0pOgogICAgICAgICIiIgogICAgICAgIERvIHRoZSB0YXNrIG9uIHRoZSBnaXZlbiBzcGVlY2ggdGltZXN0YW1wcy4gVGhlIHRhc2sgd2lsbCBkaWFyaXplIHRoZSBWQUQgc3BlZWNoIHRpbWVzdGFtcHMgaW50byBzcGVha2Vycy4KCiAgICAgICAgOnBhcmFtIHNwZWVjaF90aW1lc3RhbXBzOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgcGVyIGNoYW5uZWwgdG8gZG8gdGhlIHRhc2sgb24gYXMgb3V0cHV0dGVkIGZyb20gdGhlIFZBRC4KICAgICAgICAiIiIKICAgICAgICAjIEdldCB0aGUgc3BlYWtlciBsYWJlbHMgKHNldCBkZWZhdWx0IGlmIG5vdCBnaXZlbik6CiAgICAgICAgc3BlYWtlcl9sYWJlbHMgPSBzZWxmLl9zcGVha2VyX2xhYmVscyBvciBbCiAgICAgICAgICAgIGYic3BlYWtlcl97aX0iIGZvciBpIGluIHJhbmdlKGxlbihzcGVlY2hfdGltZXN0YW1wcykpCiAgICAgICAgXQoKICAgICAgICAjIERpYXJpemUgLSBvcmdhbml6ZSB0aGUgc3BlZWNoIHRpbWVzdGFtcHMgaW50byBhIHNpbmdsZSBsaXN0IG9mIHNwZWFrZXJzIGFuZCBzb3J0IGl0IGJ5IHN0YXJ0IHRpbWU6CiAgICAgICAgc3BlZWNoX2RpYXJpemF0aW9uID0gWwogICAgICAgICAgICAoc3BlZWNoX3RpbWVzdGFtcFsic3RhcnQiXSwgc3BlZWNoX3RpbWVzdGFtcFsiZW5kIl0sIHNwZWFrZXJfbGFiZWwpCiAgICAgICAgICAgIGZvciBzcGVha2VyX2xhYmVsLCBjaGFubmVsX3NwZWVjaF90aW1lc3RhbXBzIGluIHppcCgKICAgICAgICAgICAgICAgIHNwZWFrZXJfbGFiZWxzLCBzcGVlY2hfdGltZXN0YW1wcwogICAgICAgICAgICApCiAgICAgICAgICAgIGZvciBzcGVlY2hfdGltZXN0YW1wIGluIGNoYW5uZWxfc3BlZWNoX3RpbWVzdGFtcHMKICAgICAgICBdCiAgICAgICAgc3BlZWNoX2RpYXJpemF0aW9uLnNvcnQoKQogICAgICAgIHNlbGYuX3Jlc3VsdCA9IHNwZWVjaF9kaWFyaXphdGlvbgoKICAgIGRlZiB0b190dXBsZShzZWxmKSAtPiBUdXBsZVtzdHIsIGRpY3RdOgogICAgICAgICIiIgogICAgICAgIENvbnZlcnQgdGhlIHRhc2sgdG8gYSB0dXBsZSB0byByZWNvbnN0cnVjdCBpdCBsYXRlciAodXNlZCBmb3IgbXVsdGlwcm9jZXNzaW5nIHRvIHBhc3MgaW4gcXVldWUpLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNvbnZlcnRlZCB0YXNrLgogICAgICAgICIiIgogICAgICAgIHRhc2tfY2xhc3MsIHRhc2tfa3dhcmdzID0gc3VwZXIoKS50b190dXBsZSgpCiAgICAgICAgcmV0dXJuIHRhc2tfY2xhc3MsIHsqKnRhc2tfa3dhcmdzLCAic3BlYWtlcl9sYWJlbHMiOiBzZWxmLl9zcGVha2VyX2xhYmVsc30KCgpjbGFzcyBUYXNrQ3JlYXRvcjoKICAgICIiIgogICAgQSB0YXNrIGNyZWF0b3IgdG8gY3JlYXRlIGRpZmZlcmVudCB0YXNrcyB0byBydW4gYWZ0ZXIgdGhlIFZBRC4KICAgICIiIgoKICAgICM6IEEgbWFwIGZyb20gdGFzayBjbGFzcyBuYW1lIHRvIHRhc2sgY2xhc3MgdG8gdXNlIGluIGBmcm9tX3R1cGxlYDoKICAgIF9NQVAgPSB7CiAgICAgICAgQmFzZVRhc2suX19uYW1lX186IEJhc2VUYXNrLAogICAgICAgIFNwZWVjaERpYXJpemF0aW9uVGFzay5fX25hbWVfXzogU3BlZWNoRGlhcml6YXRpb25UYXNrLAogICAgfQoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCB0YXNrX3R5cGU6IFR5cGVbQmFzZVRhc2tdLCB0YXNrX2t3YXJnczogZGljdCA9IE5vbmUpOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgdGhlIHRhc2sgY3JlYXRvci4KICAgICAgICA6cGFyYW0gdGFza190eXBlOiBUaGUgdGFzayB0eXBlIC0gYSBgQmFzZVRhc2tgIHN1YmNsYXNzLgogICAgICAgIDpwYXJhbSB0YXNrX2t3YXJnczogQWRkaXRpb25hbCBrZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHRvIHRoZSB0byBiZSBjcmVhdGVkIHRhc2tzLgogICAgICAgICIiIgogICAgICAgIHNlbGYuX3Rhc2tfdHlwZSA9IHRhc2tfdHlwZQogICAgICAgIHNlbGYuX3Rhc2tfa3dhcmdzID0gdGFza19rd2FyZ3Mgb3Ige30KCiAgICBkZWYgY3JlYXRlX3Rhc2soc2VsZiwgYXVkaW9fZmlsZTogUGF0aCkgLT4gQmFzZVRhc2s6CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGEgdGFzayB3aXRoIHRoZSBnaXZlbiBhdWRpbyBmaWxlLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogVGhlIGF1ZGlvIGZpbGUgdG8gYXNzaWduIHRvIHRoZSB0YXNrLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNyZWF0ZWQgdGFzay4KICAgICAgICAiIiIKICAgICAgICByZXR1cm4gc2VsZi5fdGFza190eXBlKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSwgKipzZWxmLl90YXNrX2t3YXJncykKCiAgICBAY2xhc3NtZXRob2QKICAgIGRlZiBmcm9tX3R1cGxlKGNscywgdGFza190dXBsZTogVHVwbGVbc3RyLCBkaWN0XSkgLT4gQmFzZVRhc2s6CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGEgdGFzayBmcm9tIGEgdHVwbGUgb2YgdGhlIGF1ZGlvIGZpbGUgbmFtZSBhbmQgdGhlIHRhc2sga3dhcmdzLgoKICAgICAgICA6cGFyYW0gdGFza190dXBsZTogVGhlIHRhc2sgdHVwbGUgdG8gY3JlYXRlIHRoZSB0YXNrIGZyb20uCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY3JlYXRlZCB0YXNrLgogICAgICAgICIiIgogICAgICAgIHRhc2tfY2xhc3MsIHRhc2tfa3dhcmdzID0gdGFza190dXBsZQogICAgICAgIHJldHVybiBjbHMuX01BUFt0YXNrX2NsYXNzXSgqKnRhc2tfa3dhcmdzKQoKCmNsYXNzIFZvaWNlQWN0aXZpdHlEZXRlY3RvcjoKICAgICIiIgogICAgQSB2b2ljZSBhY3Rpdml0eSBkZXRlY3Rpb24gd3JhcHBlciBmb3IgdGhlIHNpbGVybyBWQUQgbW9kZWwgLSBodHRwczovL2dpdGh1Yi5jb20vc25ha2VyczQvc2lsZXJvLXZhZC4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgICMgTW9kZWwgbG9hZGluZyBrd2FyZ3M6CiAgICAgICAgdXNlX29ubng6IGJvb2wgPSBUcnVlLAogICAgICAgIGZvcmNlX29ubnhfY3B1OiBib29sID0gVHJ1ZSwKICAgICAgICAjIERldGVjdGlvbiBrd2FyZ3M6CiAgICAgICAgdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgICAgICBzYW1wbGluZ19yYXRlOiBpbnQgPSAxNl8wMDAsCiAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tczogaW50ID0gMjUwLAogICAgICAgIG1heF9zcGVlY2hfZHVyYXRpb25fczogZmxvYXQgPSBmbG9hdCgiaW5mIiksCiAgICAgICAgbWluX3NpbGVuY2VfZHVyYXRpb25fbXM6IGludCA9IDEwMCwKICAgICAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzOiBpbnQgPSA1MTIsCiAgICAgICAgc3BlZWNoX3BhZF9tczogaW50ID0gMzAsCiAgICAgICAgcmV0dXJuX3NlY29uZHM6IGJvb2wgPSBGYWxzZSwKICAgICAgICBwZXJfY2hhbm5lbDogYm9vbCA9IEZhbHNlLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSB2b2ljZSBhY3Rpdml0eSBkZXRlY3Rvci4KCiAgICAgICAgOnBhcmFtIHVzZV9vbm54OiAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHVzZSBPTk5YIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgICAgICA6cGFyYW0gZm9yY2Vfb25ueF9jcHU6ICAgICAgICAgIFdoZXRoZXIgdG8gZm9yY2UgT05OWCB0byB1c2UgQ1BVIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgICAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHByb2JhYmlsaXRpZXMgQUJPVkUgdGhpcyB2YWx1ZSBhcmUgY29uc2lkZXJlZCBhcyBTUEVFQ0guIEl0IGlzIGJldHRlciB0byB0dW5lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGlzIHBhcmFtZXRlciBmb3IgZWFjaCBkYXRhc2V0IHNlcGFyYXRlbHksIGJ1dCAibGF6eSIgMC41IGlzIHByZXR0eSBnb29kIGZvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgICAgICA6cGFyYW0gc2FtcGxpbmdfcmF0ZTogICAgICAgICAgIEN1cnJlbnRseSwgc2lsZXJvIFZBRCBtb2RlbHMgc3VwcG9ydCA4MDAwIGFuZCAxNjAwMCBzYW1wbGUgcmF0ZXMuCiAgICAgICAgOnBhcmFtIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6ICBGaW5hbCBzcGVlY2ggY2h1bmtzIHNob3J0ZXIgbWluX3NwZWVjaF9kdXJhdGlvbl9tcyBhcmUgdGhyb3duIG91dC4KICAgICAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtYXhfc3BlZWNoX2R1cmF0aW9uX3NgIHdpbGwgYmUgc3BsaXQgYXQgdGhlIHRpbWVzdGFtcCBvZiB0aGUgbGFzdCBzaWxlbmNlIHRoYXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxhc3RzIG1vcmUgdGhhbiAxMDBtcyAoaWYgYW55KSwgdG8gcHJldmVudCBhZ2dyZXNzaXZlIGN1dHRpbmcuIE90aGVyd2lzZSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXkgd2lsbCBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgICAgIDpwYXJhbSBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogSW4gdGhlIGVuZCBvZiBlYWNoIHNwZWVjaCBjaHVuayB3YWl0IGZvciBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyBiZWZvcmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNlcGFyYXRpbmcgaXQuCiAgICAgICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgcmF0ZSBhbmQgMjU2LCA1MTIsIDc2OCBzYW1wbGVzIGZvciA4MDAwIHNhbXBsZSByYXRlLiBWYWx1ZXMgb3RoZXIgdGhhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlc2UgbWF5IGFmZmVjdCBtb2RlbCBwZXJmb3JtYW5jZSEKICAgICAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgICAgICA6cGFyYW0gcmV0dXJuX3NlY29uZHM6ICAgICAgICAgIFdoZXRoZXIgcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2Vjb25kcy4gRmFsc2UgbWVhbnMgdG8gcmV0dXJuIHRpbWVzdGFtcHMgaW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZXMgKGRlZmF1bHQgLSBGYWxzZSkuCiAgICAgICAgOnBhcmFtIHBlcl9jaGFubmVsOiAgICAgICAgICAgICBXaGV0aGVyIHRvIHJldHVybiB0aW1lc3RhbXBzIHBlciBjaGFubmVsIChkZWZhdWx0IC0gRmFsc2UpLiBUaGlzIHdpbGwgcnVuIFZBRAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb24gZWFjaCBjaGFubmVsIHNlcGFyYXRlbHkgYW5kIHJldHVybiBhIGxpc3Qgb2YgdGltZXN0YW1wcyBwZXIgY2hhbm5lbC4KICAgICAgICAiIiIKICAgICAgICAjIFN0b3JlIGNvbmZpZ3VyYXRpb25zOgogICAgICAgIHNlbGYuX3VzZV9vbm54ID0gdXNlX29ubngKICAgICAgICBzZWxmLl9mb3JjZV9vbm54X2NwdSA9IGZvcmNlX29ubnhfY3B1CiAgICAgICAgc2VsZi5fdGhyZXNob2xkID0gdGhyZXNob2xkCiAgICAgICAgc2VsZi5fc2FtcGxpbmdfcmF0ZSA9IHNhbXBsaW5nX3JhdGUKICAgICAgICBzZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zID0gbWluX3NwZWVjaF9kdXJhdGlvbl9tcwogICAgICAgIHNlbGYuX21heF9zcGVlY2hfZHVyYXRpb25fcyA9IG1heF9zcGVlY2hfZHVyYXRpb25fcwogICAgICAgIHNlbGYuX21pbl9zaWxlbmNlX2R1cmF0aW9uX21zID0gbWluX3NpbGVuY2VfZHVyYXRpb25fbXMKICAgICAgICBzZWxmLl93aW5kb3dfc2l6ZV9zYW1wbGVzID0gd2luZG93X3NpemVfc2FtcGxlcwogICAgICAgIHNlbGYuX3NwZWVjaF9wYWRfbXMgPSBzcGVlY2hfcGFkX21zCiAgICAgICAgc2VsZi5fcmV0dXJuX3NlY29uZHMgPSByZXR1cm5fc2Vjb25kcwogICAgICAgIHNlbGYuX3Blcl9jaGFubmVsID0gcGVyX2NoYW5uZWwKCiAgICAgICAgIyBQcmVwYXJlIHRoZSBtb2RlbCB2YXJpYWJsZXMKICAgICAgICBzZWxmLl9tb2RlbDogdG9yY2guTW9kdWxlID0gTm9uZQogICAgICAgIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wczogRnVuY3Rpb25UeXBlID0gTm9uZQoKICAgIGRlZiBsb2FkKHNlbGYsIGZvcmNlX3JlbG9hZDogYm9vbCA9IFRydWUpOgogICAgICAgICIiIgogICAgICAgIExvYWQgdGhlIFZBRCBtb2RlbC4KCiAgICAgICAgOnBhcmFtIGZvcmNlX3JlbG9hZDogV2hldGhlciB0byBmb3JjZSByZWxvYWQgdGhlIG1vZGVsIGV2ZW4gaWYgaXQgd2FzIGFscmVhZHkgbG9hZGVkLiBEZWZhdWx0IGlzIFRydWUuCiAgICAgICAgIiIiCiAgICAgICAgbW9kZWwsIHV0aWxzID0gdG9yY2guaHViLmxvYWQoCiAgICAgICAgICAgIHJlcG9fb3JfZGlyPSJzbmFrZXJzNC9zaWxlcm8tdmFkIiwKICAgICAgICAgICAgbW9kZWw9InNpbGVyb192YWQiLAogICAgICAgICAgICBmb3JjZV9yZWxvYWQ9Zm9yY2VfcmVsb2FkLAogICAgICAgICAgICBvbm54PXNlbGYuX3VzZV9vbm54LAogICAgICAgICAgICBmb3JjZV9vbm54X2NwdT1zZWxmLl9mb3JjZV9vbm54X2NwdSwKICAgICAgICApCiAgICAgICAgc2VsZi5fbW9kZWwgPSBtb2RlbAogICAgICAgICgKICAgICAgICAgICAgc2VsZi5fZ2V0X3NwZWVjaF90aW1lc3RhbXBzLAogICAgICAgICAgICBfLCAgIyBzYXZlX2F1ZGlvLAogICAgICAgICAgICBfLCAgIyByZWFkX2F1ZGlvLAogICAgICAgICAgICBfLCAgIyBWQURJdGVyYXRvciwKICAgICAgICAgICAgXywgICMgY29sbGVjdF9jaHVua3MKICAgICAgICApID0gdXRpbHMKCiAgICBkZWYgZGV0ZWN0X3ZvaWNlKAogICAgICAgIHNlbGYsCiAgICAgICAgYXVkaW9fZmlsZTogUGF0aCwKICAgICkgLT4gVW5pb25bTGlzdFtEaWN0W3N0ciwgaW50XV0sIExpc3RbTGlzdFtEaWN0W3N0ciwgaW50XV1dXToKICAgICAgICAiIiIKICAgICAgICBJbmZlciB0aGUgYXVkaW8gdGhyb3VnaCB0aGUgVkFEIG1vZGVsIGFuZCByZXR1cm4gdGhlIHNwZWVjaCB0aW1lc3RhbXBzLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogVGhlIGF1ZGlvIGZpbGUgdG8gaW5mZXIuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgaW4gdGhlIGF1ZGlvLiBBIGxpc3Qgb2YgdGltZXN0YW1wcyB3aGVyZSBlYWNoIHRpbWVzdGFtcCBpcyBhIGRpY3Rpb25hcnkgd2l0aCB0aGUKICAgICAgICAgICAgICAgICBmb2xsb3dpbmcga2V5czoKCiAgICAgICAgICAgICAgICAgKiAic3RhcnQiOiBUaGUgc3RhcnQgc2FtcGxlIGluZGV4IG9mIHRoZSBzcGVlY2ggaW4gdGhlIGF1ZGlvLgogICAgICAgICAgICAgICAgICogImVuZCI6ICAgVGhlIGVuZCBzYW1wbGUgaW5kZXggb2YgdGhlIHNwZWVjaCBpbiB0aGUgYXVkaW8uCgogICAgICAgICAgICAgICAgIElmIGBwZXJfY2hhbm5lbGAgaXMgVHJ1ZSwgYSBsaXN0IG9mIHRpbWVzdGFtcHMgcGVyIGNoYW5uZWwgd2lsbCBiZSByZXR1cm5lZC4KICAgICAgICAiIiIKICAgICAgICAjIENhc3QgdG8gYSBudW1weSBhcnJheToKICAgICAgICBhdWRpbyA9IHNlbGYuX3JlYWRfYXVkaW8oYXVkaW9fZmlsZSkKCiAgICAgICAgIyBEZXRlY3Qgc3BlZWNoOgogICAgICAgIGlmIG5vdCBzZWxmLl9wZXJfY2hhbm5lbDoKICAgICAgICAgICAgcmV0dXJuIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wcygKICAgICAgICAgICAgICAgIGF1ZGlvLAogICAgICAgICAgICAgICAgc2VsZi5fbW9kZWwsCiAgICAgICAgICAgICAgICB0aHJlc2hvbGQ9c2VsZi5fdGhyZXNob2xkLAogICAgICAgICAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tcz1zZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zPXNlbGYuX21heF9zcGVlY2hfZHVyYXRpb25fcywKICAgICAgICAgICAgICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zPXNlbGYuX21pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgc3BlZWNoX3BhZF9tcz1zZWxmLl9zcGVlY2hfcGFkX21zLAogICAgICAgICAgICAgICAgc2FtcGxpbmdfcmF0ZT1zZWxmLl9zYW1wbGluZ19yYXRlLAogICAgICAgICAgICAgICAgd2luZG93X3NpemVfc2FtcGxlcz1zZWxmLl93aW5kb3dfc2l6ZV9zYW1wbGVzLAogICAgICAgICAgICAgICAgcmV0dXJuX3NlY29uZHM9c2VsZi5fcmV0dXJuX3NlY29uZHMsCiAgICAgICAgICAgICkKCiAgICAgICAgIyBQZXIgY2hhbm5lbDoKICAgICAgICBzcGVlY2hfdGltZXN0YW1wcyA9IFtdCiAgICAgICAgZm9yIGNoYW5uZWwgaW4gYXVkaW86CiAgICAgICAgICAgIHNwZWVjaF90aW1lc3RhbXBzLmFwcGVuZCgKICAgICAgICAgICAgICAgIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wcygKICAgICAgICAgICAgICAgICAgICBjaGFubmVsLAogICAgICAgICAgICAgICAgICAgIHNlbGYuX21vZGVsLAogICAgICAgICAgICAgICAgICAgIHRocmVzaG9sZD1zZWxmLl90aHJlc2hvbGQsCiAgICAgICAgICAgICAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tcz1zZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgICAgIG1heF9zcGVlY2hfZHVyYXRpb25fcz1zZWxmLl9tYXhfc3BlZWNoX2R1cmF0aW9uX3MsCiAgICAgICAgICAgICAgICAgICAgbWluX3NpbGVuY2VfZHVyYXRpb25fbXM9c2VsZi5fbWluX3NpbGVuY2VfZHVyYXRpb25fbXMsCiAgICAgICAgICAgICAgICAgICAgc3BlZWNoX3BhZF9tcz1zZWxmLl9zcGVlY2hfcGFkX21zLAogICAgICAgICAgICAgICAgICAgIHNhbXBsaW5nX3JhdGU9c2VsZi5fc2FtcGxpbmdfcmF0ZSwKICAgICAgICAgICAgICAgICAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzPXNlbGYuX3dpbmRvd19zaXplX3NhbXBsZXMsCiAgICAgICAgICAgICAgICAgICAgcmV0dXJuX3NlY29uZHM9c2VsZi5fcmV0dXJuX3NlY29uZHMsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKCiAgICAgICAgcmV0dXJuIHNwZWVjaF90aW1lc3RhbXBzCgogICAgZGVmIF9yZWFkX2F1ZGlvKAogICAgICAgIHNlbGYsCiAgICAgICAgcGF0aDogUGF0aCwKICAgICkgLT4gdG9yY2guVGVuc29yOgogICAgICAgICIiIgogICAgICAgIFJlYWQgdGhlIGF1ZGlvIGZyb20gdGhlIGdpdmVuIHBhdGggYW5kIHJldHVybiBpdCBhcyBhIHRlbnNvci4KCiAgICAgICAgOnBhcmFtIHBhdGg6IFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlLgoKICAgICAgICA6cmV0dXJuczogVGhlIGF1ZGlvIGFzIGEgdGVuc29yLgogICAgICAgICIiIgogICAgICAgICMgUmVhZCB0aGUgYXVkaW86CiAgICAgICAgYXVkaW8sIHNhbXBsaW5nX3JhdGUgPSB0b3JjaGF1ZGlvLmxvYWQoc3RyKHBhdGgpKQoKICAgICAgICAjIENoZWNrIGlmIHRoZSBhdWRpbyBpcyBzdGVyZW8gYW5kIGlmIHNvLCBjb252ZXJ0IGl0IHRvIG1vbm8gKG9ubHkgaWYgbm90IHBlciBjaGFubmVsKToKICAgICAgICBpZiBhdWRpby5zaXplKDApID4gMSBhbmQgbm90IHNlbGYuX3Blcl9jaGFubmVsOgogICAgICAgICAgICBhdWRpbyA9IGF1ZGlvLm1lYW4oZGltPTAsIGtlZXBkaW09VHJ1ZSkKCiAgICAgICAgIyBSZXNhbXBsZSB0aGUgYXVkaW8gaWYgbmVlZGVkOgogICAgICAgIGlmIHNhbXBsaW5nX3JhdGUgIT0gc2VsZi5fc2FtcGxpbmdfcmF0ZToKICAgICAgICAgICAgdHJhbnNmb3JtID0gdG9yY2hhdWRpby50cmFuc2Zvcm1zLlJlc2FtcGxlKAogICAgICAgICAgICAgICAgb3JpZ19mcmVxPXNhbXBsaW5nX3JhdGUsIG5ld19mcmVxPXNlbGYuX3NhbXBsaW5nX3JhdGUKICAgICAgICAgICAgKQogICAgICAgICAgICBhdWRpbyA9IHRyYW5zZm9ybShhdWRpbykKCiAgICAgICAgIyBSZXR1cm4gdGhlIGF1ZGlvIChzcXVlZXplIGlmIG5vdCBwZXIgY2hhbm5lbCk6CiAgICAgICAgcmV0dXJuIGF1ZGlvIGlmIHNlbGYuX3Blcl9jaGFubmVsIGVsc2UgYXVkaW8uc3F1ZWV6ZSgwKQoKCiM6IFRoZSB2YWx1ZSB0byBzZW5kIGludG8gbXVsdGlwcm9jZXNzaW5nIHF1ZXVlcyB0byBzdG9wIHRoZSBwcm9jZXNzOgpfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSyA9ICJTVE9QIgoKCmRlZiBfbXVsdGlwcm9jZXNzaW5nX2NvbXBsZXRlX3Rhc2tzKAogICAgdmFkX2luaXRfa3dhcmdzOiBkaWN0LCB0YXNrc19xdWV1ZTogUXVldWUsIHJlc3VsdHNfcXVldWU6IFF1ZXVlCik6CiAgICAiIiIKICAgIENvbXBsZXRlIHRoZSB0YXNrcyBpbiB0aGUgZ2l2ZW4gcXVldWUgYW5kIHB1dCB0aGUgcmVzdWx0cyBpbiB0aGUgZ2l2ZW4gcmVzdWx0cyBxdWV1ZS4gVGhlIGZ1bmN0aW9uIHdpbGwgc3RvcCB3aGVuCiAgICB0aGUgZ2l2ZW4gdGFza3MgcXVldWUgd2lsbCByZWNlaXZlIHRoZSBzdG9wIG1hcmsuIEl0IGlzIGFpbWVkIHRvIGJlIHVzZWQgd2l0aCBtdWx0aXByb2Nlc3NpbmcgYXMgYSBwcm9jZXNzLgoKICAgIDpwYXJhbSB2YWRfaW5pdF9rd2FyZ3M6IFRoZSBWQUQgaW5pdGlhbGl6YXRpb24ga3dhcmdzLgogICAgOnBhcmFtIHRhc2tzX3F1ZXVlOiAgICAgQSBxdWV1ZSB0byBnZXQgdGhlIHRhc2tzIGZyb20uCiAgICA6cGFyYW0gcmVzdWx0c19xdWV1ZTogICBBIHF1ZXVlIHRvIHB1dCB0aGUgcmVzdWx0cyBpbi4KICAgICIiIgogICAgIyBJbml0aWFsaXplIGFuZCBsb2FkIHRoZSBWQUQ6CiAgICB2YWQgPSBWb2ljZUFjdGl2aXR5RGV0ZWN0b3IoKip2YWRfaW5pdF9rd2FyZ3MpCiAgICB2YWQubG9hZChmb3JjZV9yZWxvYWQ9RmFsc2UpCgogICAgIyBTdGFydCBsaXN0ZW5pbmcgdG8gdGhlIHRhc2tzIHF1ZXVlOgogICAgd2hpbGUgVHJ1ZToKICAgICAgICAjIEdldCB0aGUgdGFzazoKICAgICAgICB0YXNrOiBUdXBsZVtzdHIsIGRpY3RdID0gdGFza3NfcXVldWUuZ2V0KCkKICAgICAgICBpZiB0YXNrID09IF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLOgogICAgICAgICAgICBicmVhawogICAgICAgIHRyeToKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2sgPSBUYXNrQ3JlYXRvci5mcm9tX3R1cGxlKHRhc2tfdHVwbGU9dGFzaykKICAgICAgICAgICAgIyBSdW4gdGhlIGZpbGUgdGhyb3VnaCB0aGUgVkFEOgogICAgICAgICAgICBzcGVlY2hfdGltZXN0YW1wcyA9IHZhZC5kZXRlY3Rfdm9pY2UoYXVkaW9fZmlsZT10YXNrLmF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBCdWlsZCB0aGUgcmVzdWx0OgogICAgICAgICAgICByZXN1bHQgPSAoRmFsc2UsIHRhc2suZ2V0X3Jlc3VsdCgpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIEJ1aWxkIHRoZSBlcnJvcjoKICAgICAgICAgICAgcmVzdWx0ID0gKFRydWUsICh0YXNrLmF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpKQogICAgICAgICMgQ29sbGVjdCB0aGUgcmVzdWx0IC8gZXJyb3I6CiAgICAgICAgcmVzdWx0c19xdWV1ZS5wdXQocmVzdWx0KQoKICAgICMgTWFyayB0aGUgZW5kIG9mIHRoZSB0YXNrczoKICAgIHJlc3VsdHNfcXVldWUucHV0KF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLKQoKCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgp0cnk6CiAgICBpbXBvcnQgbWxydW4KCiAgICBfTE9HR0VSID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgoInNpbGVyb192YWQiKS5sb2dnZXIKZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICBfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBkZXRlY3Rfdm9pY2UoCiAgICAjIElucHV0IGt3YXJnczoKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBQYXRoLCBMaXN0W1VuaW9uW3N0ciwgUGF0aF1dXSwKICAgICMgTW9kZWwgbG9hZGluZyBrd2FyZ3M6CiAgICB1c2Vfb25ueDogYm9vbCA9IFRydWUsCiAgICBmb3JjZV9vbm54X2NwdTogYm9vbCA9IFRydWUsCiAgICAjIERldGVjdGlvbiBrd2FyZ3M6CiAgICB0aHJlc2hvbGQ6IGZsb2F0ID0gMC41LAogICAgc2FtcGxpbmdfcmF0ZTogaW50ID0gMTZfMDAwLAogICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tczogaW50ID0gMjUwLAogICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiBmbG9hdCA9IGZsb2F0KCJpbmYiKSwKICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBpbnQgPSAxMDAsCiAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzOiBpbnQgPSA1MTIsCiAgICBzcGVlY2hfcGFkX21zOiBpbnQgPSAzMCwKICAgIHJldHVybl9zZWNvbmRzOiBib29sID0gRmFsc2UsCiAgICBwZXJfY2hhbm5lbDogYm9vbCA9IEZhbHNlLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHZvaWNlIGFjdGl2aXR5IGRldGVjdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtCiAgICBodHRwczovL2dpdGh1Yi5jb20vc25ha2VyczQvc2lsZXJvLXZhZC4gVGhlIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIKICAgIFZBRCB0aW1lc3RhbXBzIGRpY3Rpb25hcmllcyBhcyB2YWx1ZS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICB7InN0YXJ0IjogMCwgImVuZCI6IDE2MDAwfSwKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAxNjAwMCwgImVuZCI6IDMyMDAwfSwKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAzMjAwMCwgImVuZCI6IDQ4MDAwfSwKICAgICAgICAgICAgICAgIC4uLgogICAgICAgICAgICBdLAogICAgICAgICAgICAiZmlsZV8yLndhdiI6IFsKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAwLCAiZW5kIjogMTYwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDE2MDAwLCAiZW5kIjogMzIwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDMyMDAwLCAiZW5kIjogNDgwMDB9LAogICAgICAgICAgICAgICAgLi4uCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgIC4uLgogICAgICAgIH0KCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgICAgICBUaGUgcGF0aCB0byB0aGUgYXVkaW8gZmlsZXMgdG8gZGlhcml6ZS4gQ2FuIGJlIGEgcGF0aCB0byBhIHNpbmdsZSBmaWxlLCBhIHBhdGggdG8gYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkaXJlY3Rvcnkgb3IgYSBsaXN0IG9mIHBhdGhzIHRvIGZpbGVzLgogICAgOnBhcmFtIHVzZV9vbm54OiAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHVzZSBPTk5YIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgIDpwYXJhbSBmb3JjZV9vbm54X2NwdTogICAgICAgICAgV2hldGhlciB0byBmb3JjZSBPTk5YIHRvIHVzZSBDUFUgZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIHRocmVzaG9sZDogICAgICAgICAgICAgICBTcGVlY2ggdGhyZXNob2xkLiBTaWxlcm8gVkFEIG91dHB1dHMgc3BlZWNoIHByb2JhYmlsaXRpZXMgZm9yIGVhY2ggYXVkaW8gY2h1bmssCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHByb2JhYmlsaXRpZXMgQUJPVkUgdGhpcyB2YWx1ZSBhcmUgY29uc2lkZXJlZCBhcyBTUEVFQ0guIEl0IGlzIGJldHRlciB0byB0dW5lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoaXMgcGFyYW1ldGVyIGZvciBlYWNoIGRhdGFzZXQgc2VwYXJhdGVseSwgYnV0ICJsYXp5IiAwLjUgaXMgcHJldHR5IGdvb2QgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vc3QgZGF0YXNldHMuCiAgICA6cGFyYW0gc2FtcGxpbmdfcmF0ZTogICAgICAgICAgIEN1cnJlbnRseSwgc2lsZXJvIFZBRCBtb2RlbHMgc3VwcG9ydCA4MDAwIGFuZCAxNjAwMCBzYW1wbGUgcmF0ZXMuCiAgICA6cGFyYW0gbWluX3NwZWVjaF9kdXJhdGlvbl9tczogIEZpbmFsIHNwZWVjaCBjaHVua3Mgc2hvcnRlciBtaW5fc3BlZWNoX2R1cmF0aW9uX21zIGFyZSB0aHJvd24gb3V0LgogICAgOnBhcmFtIG1heF9zcGVlY2hfZHVyYXRpb25fczogICBNYXhpbXVtIGR1cmF0aW9uIG9mIHNwZWVjaCBjaHVua3MgaW4gc2Vjb25kcy4gQ2h1bmtzIGxvbmdlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtYXhfc3BlZWNoX2R1cmF0aW9uX3NgIHdpbGwgYmUgc3BsaXQgYXQgdGhlIHRpbWVzdGFtcCBvZiB0aGUgbGFzdCBzaWxlbmNlIHRoYXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFzdHMgbW9yZSB0aGFuIDEwMG1zIChpZiBhbnkpLCB0byBwcmV2ZW50IGFnZ3Jlc3NpdmUgY3V0dGluZy4gT3RoZXJ3aXNlLCB0aGV5IHdpbGwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYmUgc3BsaXQgYWdncmVzc2l2ZWx5IGp1c3QgYmVmb3JlIG1heF9zcGVlY2hfZHVyYXRpb25fcy4KICAgIDpwYXJhbSBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogSW4gdGhlIGVuZCBvZiBlYWNoIHNwZWVjaCBjaHVuayB3YWl0IGZvciBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyBiZWZvcmUgc2VwYXJhdGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpdC4KICAgIDpwYXJhbSB3aW5kb3dfc2l6ZV9zYW1wbGVzOiAgICAgQXVkaW8gY2h1bmtzIG9mIHdpbmRvd19zaXplX3NhbXBsZXMgc2l6ZSBhcmUgZmVkIHRvIHRoZSBzaWxlcm8gVkFEIG1vZGVsLgoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV0FSTklORyEgU2lsZXJvIFZBRCBtb2RlbHMgd2VyZSB0cmFpbmVkIHVzaW5nIDUxMiwgMTAyNCwgMTUzNiBzYW1wbGVzIGZvciAxNjAwMAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgcmF0ZSBhbmQgMjU2LCA1MTIsIDc2OCBzYW1wbGVzIGZvciA4MDAwIHNhbXBsZSByYXRlLiBWYWx1ZXMgb3RoZXIgdGhhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGVzZSBtYXkgYWZmZWN0IG1vZGVsIHBlcmZvcm1hbmNlIQogICAgOnBhcmFtIHNwZWVjaF9wYWRfbXM6ICAgICAgICAgICBGaW5hbCBzcGVlY2ggY2h1bmtzIGFyZSBwYWRkZWQgYnkgc3BlZWNoX3BhZF9tcyBlYWNoIHNpZGUuCiAgICA6cGFyYW0gcmV0dXJuX3NlY29uZHM6ICAgICAgICAgIFdoZXRoZXIgcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2Vjb25kcy4gRmFsc2UgbWVhbnMgdG8gcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2FtcGxlcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoZGVmYXVsdCAtIEZhbHNlKS4KICAgIDpwYXJhbSBwZXJfY2hhbm5lbDogICAgICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGltZXN0YW1wcyBwZXIgY2hhbm5lbCAoZGVmYXVsdCAtIEZhbHNlKS4gVGhpcyB3aWxsIHJ1biBWQUQgb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZWFjaCBjaGFubmVsIHNlcGFyYXRlbHkgYW5kIHJldHVybiBhIGxpc3Qgb2YgdGltZXN0YW1wcyBwZXIgY2hhbm5lbC4KICAgIDpwYXJhbSB1c2VfbXVsdGlwcm9jZXNzaW5nOiAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZSBmb3IgbXVsdGlwcm9jZXNzaW5nLiBJZiAwLCBubyBtdWx0aXByb2Nlc3Npbmcgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkLiBEZWZhdWx0IGlzIDAuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgICAgIFZlcmJvc2l0eS4KICAgICIiIgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIEdldCB0aGUgaW5wdXQgYXVkaW8gZmlsZXMgdG8gdHJhbnNjcmliZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJDb2xsZWN0aW5nIGF1ZGlvIGZpbGVzLiIpCiAgICBhdWRpb19maWxlcyA9IF9nZXRfYXVkaW9fZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiQ29sbGVjdGVkIHtsZW4oYXVkaW9fZmlsZXMpfSBhdWRpbyBmaWxlcy4iKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgdHJhbnNjcmlwdGlvbiBwaXBlbGluZToKICAgIHZhZF9pbml0X2t3YXJncyA9IHsKICAgICAgICAidXNlX29ubngiOiB1c2Vfb25ueCwKICAgICAgICAiZm9yY2Vfb25ueF9jcHUiOiBmb3JjZV9vbm54X2NwdSwKICAgICAgICAidGhyZXNob2xkIjogdGhyZXNob2xkLAogICAgICAgICJzYW1wbGluZ19yYXRlIjogc2FtcGxpbmdfcmF0ZSwKICAgICAgICAibWluX3NwZWVjaF9kdXJhdGlvbl9tcyI6IG1pbl9zcGVlY2hfZHVyYXRpb25fbXMsCiAgICAgICAgIm1heF9zcGVlY2hfZHVyYXRpb25fcyI6IG1heF9zcGVlY2hfZHVyYXRpb25fcywKICAgICAgICAibWluX3NpbGVuY2VfZHVyYXRpb25fbXMiOiBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcywKICAgICAgICAid2luZG93X3NpemVfc2FtcGxlcyI6IHdpbmRvd19zaXplX3NhbXBsZXMsCiAgICAgICAgInNwZWVjaF9wYWRfbXMiOiBzcGVlY2hfcGFkX21zLAogICAgICAgICJyZXR1cm5fc2Vjb25kcyI6IHJldHVybl9zZWNvbmRzLAogICAgICAgICJwZXJfY2hhbm5lbCI6IHBlcl9jaGFubmVsLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcih0YXNrX3R5cGU9QmFzZVRhc2spCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEZXRlY3Rpbmcgdm9pY2UiLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEZXRlY3Rpbmcgdm9pY2UiLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBkaWFyaXplKAogICAgIyBJbnB1dCAvIE91dHB1dCBrd2FyZ3M6CiAgICBkYXRhX3BhdGg6IFVuaW9uW3N0ciwgUGF0aCwgTGlzdFtVbmlvbltzdHIsIFBhdGhdXV0sCiAgICAjIE1vZGVsIGxvYWRpbmcga3dhcmdzOgogICAgdXNlX29ubng6IGJvb2wgPSBUcnVlLAogICAgZm9yY2Vfb25ueF9jcHU6IGJvb2wgPSBUcnVlLAogICAgIyBEZXRlY3Rpb24ga3dhcmdzOgogICAgdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgIHNhbXBsaW5nX3JhdGU6IGludCA9IDE2XzAwMCwKICAgIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6IGludCA9IDI1MCwKICAgIG1heF9zcGVlY2hfZHVyYXRpb25fczogZmxvYXQgPSBmbG9hdCgiaW5mIiksCiAgICBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogaW50ID0gMTAwLAogICAgd2luZG93X3NpemVfc2FtcGxlczogaW50ID0gNTEyLAogICAgc3BlZWNoX3BhZF9tczogaW50ID0gMzAsCiAgICAjIERpYXJpemF0aW9uIGt3YXJnczoKICAgIHNwZWFrZXJfbGFiZWxzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHNwZWVjaCBkaWFyaXphdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtIGh0dHBzOi8vZ2l0aHViLmNvbS9zbmFrZXJzNC9zaWxlcm8tdmFkLgogICAgVGhlIHNwZWVjaCBkaWFyaXphdGlvbiBpcyBwZXJmb3JtZWQgcGVyIGNoYW5uZWwgc28gdGhhdCBlYWNoIGNoYW5uZWwgaW4gdGhlIGF1ZGlvIGJlbG9uZyB0byBhIGRpZmZlcmVudCBzcGVha2VyLiBUaGUKICAgIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIgZGlhcml6YXRpb24gYXMgdmFsdWUuIEEgZGlhcml6YXRpb24gaXMgYSBsaXN0CiAgICBvZiB0dXBsZXM6IChzdGFydCwgZW5kLCBzcGVha2VyX2xhYmVsKS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgImZpbGVfMi53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgLi4uCiAgICAgICAgfQoKCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICAgICAgIFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlcyB0byBkaWFyaXplLiBDYW4gYmUgYSBwYXRoIHRvIGEgc2luZ2xlIGZpbGUsIGEgcGF0aCB0byBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRpcmVjdG9yeSBvciBhIGxpc3Qgb2YgcGF0aHMgdG8gZmlsZXMuCiAgICA6cGFyYW0gdXNlX29ubng6ICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIE9OTlggZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIGZvcmNlX29ubnhfY3B1OiAgICAgICAgICBXaGV0aGVyIHRvIGZvcmNlIE9OTlggdG8gdXNlIENQVSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIFRydWUuCiAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHJvYmFiaWxpdGllcyBBQk9WRSB0aGlzIHZhbHVlIGFyZSBjb25zaWRlcmVkIGFzIFNQRUVDSC4gSXQgaXMgYmV0dGVyIHRvIHR1bmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhpcyBwYXJhbWV0ZXIgZm9yIGVhY2ggZGF0YXNldCBzZXBhcmF0ZWx5LCBidXQgImxhenkiIDAuNSBpcyBwcmV0dHkgZ29vZCBmb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgIDpwYXJhbSBzYW1wbGluZ19yYXRlOiAgICAgICAgICAgQ3VycmVudGx5LCBzaWxlcm8gVkFEIG1vZGVscyBzdXBwb3J0IDgwMDAgYW5kIDE2MDAwIHNhbXBsZSByYXRlcy4KICAgIDpwYXJhbSBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiAgRmluYWwgc3BlZWNoIGNodW5rcyBzaG9ydGVyIG1pbl9zcGVlY2hfZHVyYXRpb25fbXMgYXJlIHRocm93biBvdXQuCiAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYG1heF9zcGVlY2hfZHVyYXRpb25fc2Agd2lsbCBiZSBzcGxpdCBhdCB0aGUgdGltZXN0YW1wIG9mIHRoZSBsYXN0IHNpbGVuY2UgdGhhdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXN0cyBtb3JlIHRoYW4gMTAwbXMgKGlmIGFueSksIHRvIHByZXZlbnQgYWdncmVzc2l2ZSBjdXR0aW5nLiBPdGhlcndpc2UsIHRoZXkgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgOnBhcmFtIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBJbiB0aGUgZW5kIG9mIGVhY2ggc3BlZWNoIGNodW5rIHdhaXQgZm9yIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zIGJlZm9yZSBzZXBhcmF0aW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGl0LgogICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSByYXRlIGFuZCAyNTYsIDUxMiwgNzY4IHNhbXBsZXMgZm9yIDgwMDAgc2FtcGxlIHJhdGUuIFZhbHVlcyBvdGhlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXNlIG1heSBhZmZlY3QgbW9kZWwgcGVyZm9ybWFuY2UhCiAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgIDpwYXJhbSBzcGVha2VyX2xhYmVsczogICAgICAgICAgVGhlIHNwZWFrZXIgbGFiZWxzIHRvIHVzZSBmb3IgdGhlIGRpYXJpemF0aW9uLiBJZiBub3QgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5hbWVkICJzcGVha2VyXzAiLCAic3BlYWtlcl8xIiwgZXRjLgogICAgOnBhcmFtIHVzZV9tdWx0aXByb2Nlc3Npbmc6ICAgICBUaGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8gdXNlIGZvciBtdWx0aXByb2Nlc3NpbmcuIElmIDAsIG5vIG11bHRpcHJvY2Vzc2luZyB3aWxsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHVzZWQuIERlZmF1bHQgaXMgMC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgVmVyYm9zaXR5LgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgYXVkaW8gZmlsZXMuIikKICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbihhdWRpb19maWxlcyl9IGF1ZGlvIGZpbGVzLiIpCgogICAgIyBJbml0aWFsaXplIHRoZSB0cmFuc2NyaXB0aW9uIHBpcGVsaW5lOgogICAgdmFkX2luaXRfa3dhcmdzID0gewogICAgICAgICJ1c2Vfb25ueCI6IHVzZV9vbm54LAogICAgICAgICJmb3JjZV9vbm54X2NwdSI6IGZvcmNlX29ubnhfY3B1LAogICAgICAgICJ0aHJlc2hvbGQiOiB0aHJlc2hvbGQsCiAgICAgICAgInNhbXBsaW5nX3JhdGUiOiBzYW1wbGluZ19yYXRlLAogICAgICAgICJtaW5fc3BlZWNoX2R1cmF0aW9uX21zIjogbWluX3NwZWVjaF9kdXJhdGlvbl9tcywKICAgICAgICAibWF4X3NwZWVjaF9kdXJhdGlvbl9zIjogbWF4X3NwZWVjaF9kdXJhdGlvbl9zLAogICAgICAgICJtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyI6IG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICJ3aW5kb3dfc2l6ZV9zYW1wbGVzIjogd2luZG93X3NpemVfc2FtcGxlcywKICAgICAgICAic3BlZWNoX3BhZF9tcyI6IHNwZWVjaF9wYWRfbXMsCiAgICAgICAgInJldHVybl9zZWNvbmRzIjogVHJ1ZSwKICAgICAgICAicGVyX2NoYW5uZWwiOiBUcnVlLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcigKICAgICAgICB0YXNrX3R5cGU9U3BlZWNoRGlhcml6YXRpb25UYXNrLCB0YXNrX2t3YXJncz17InNwZWFrZXJfbGFiZWxzIjogc3BlYWtlcl9sYWJlbHN9CiAgICApCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgZGF0YV9wYXRoOiBVbmlvbltQYXRoLCBzdHIsIGxpc3RdLAopIC0+IExpc3RbUGF0aF06CiAgICAiIiIKICAgIEdldCB0aGUgYXVkaW8gZmlsZXMgZnJvbSB0aGUgZGF0YSBwYXRoLiBJZiBhIHBhdGggdG8gYSBkaXJlY3RvcnkgaXMgZ2l2ZW4sIGFsbCBmaWxlcyBpbiB0aGUgZGlyZWN0b3J5IHdpbGwgYmUKICAgIGNvbGxlY3RlZC4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiBUaGUgZGF0YSBwYXRoIHRvIGNvbGxlY3QgdGhlIGF1ZGlvIGZpbGVzIGZyb20uCgogICAgOnJldHVybnM6IFRoZSBhdWRpbyBmaWxlcyBsaXN0LgogICAgIiIiCiAgICAjIENoZWNrIGlmIGdpdmVuIGEgbGlzdCBvZiBwYXRoczoKICAgIGlmIGlzaW5zdGFuY2UoZGF0YV9wYXRoLCBsaXN0KToKICAgICAgICBhdWRpb19maWxlcyA9IFtdCiAgICAgICAgZm9yIHBhdGggaW4gZGF0YV9wYXRoOgogICAgICAgICAgICBhdWRpb19maWxlcy5leHRlbmQoX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9cGF0aCkpCiAgICAgICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgogICAgIyBDaGVjayBpZiBnaXZlbiBhIHNpbmdsZSBzdHJpbmcgcGF0aCB0byBjYXN0IGl0IHRvIGEgYHBhdGhsaWIuUGF0aGA6CiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgc3RyKToKICAgICAgICBkYXRhX3BhdGggPSBQYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQoKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICBhdWRpb19maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gW2RhdGFfcGF0aF0KICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJVbnJlY29nbml6ZWQgZGF0YSBwYXRoLiBUaGUgcGFyYW1ldGVyIGBkYXRhX3BhdGhgIG11c3QgYmUgYSB2YWxpZCBwYXRoIHRvIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgIgogICAgICAgICAgICBmImZpbGUuIEdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgoKZGVmIF9ydW4oCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgTG9hZCBhIFZBRCBhbmQgdXNlIGl0IHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcgdGhlIGdpdmVuIHRhc2sgY3JlYXRvci4KCiAgICA6cGFyYW0gYXVkaW9fZmlsZXM6ICAgICBUaGUgYXVkaW8gZmlsZXMgdG8gdXNlLgogICAgOnBhcmFtIGRlc2NyaXB0aW9uOiAgICAgVGhlIGRlc2NyaXB0aW9uIHRvIHVzZSBmb3IgdGhlIHByb2dyZXNzIGJhci4KICAgIDpwYXJhbSB2YWRfaW5pdF9rd2FyZ3M6IFRoZSBWQUQgaW5pdGlhbGl6YXRpb24ga2V5d29yZCBhcmd1bWVudHMuCiAgICA6cGFyYW0gdGFza19jcmVhdG9yOiAgICBUaGUgdGFzayBjcmVhdG9yIHRvIHVzZSB0byBjcmVhdGUgdGhlIHRhc2tzLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgVmVyYm9zaXR5LgoKICAgIDpyZXR1cm5zOiBUaGUgY29sbGVjdGVkIHJlc3VsdHMuCiAgICAiIiIKICAgICMgTG9hZCB0aGUgVkFEOgogICAgdmFkID0gVm9pY2VBY3Rpdml0eURldGVjdG9yKCoqdmFkX2luaXRfa3dhcmdzKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJMb2FkaW5nIHRoZSBWQUQgbW9kZWwuIikKICAgIHZhZC5sb2FkKCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJWQUQgbW9kZWwgbG9hZGVkLiIpCgogICAgIyBSdW4gdGhlIFZBRCBvbiB0aGUgYXVkaW8gZmlsZXMgYW5kIGNvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsCiAgICAgICAgZGVzYz1kZXNjcmlwdGlvbiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICAgICB0b3RhbD1sZW4oYXVkaW9fZmlsZXMpLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2sgPSB0YXNrX2NyZWF0b3IuY3JlYXRlX3Rhc2soYXVkaW9fZmlsZT1hdWRpb19maWxlKQogICAgICAgICAgICAjIFJ1biB0aGUgZmlsZSB0aHJvdWdoIHRoZSBWQUQ6CiAgICAgICAgICAgIHNwZWVjaF90aW1lc3RhbXBzID0gdmFkLmRldGVjdF92b2ljZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSByZXN1bHQ6CiAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKChGYWxzZSwgdGFzay5nZXRfcmVzdWx0KCkpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIENvbGxlY3QgdGhlIGVycm9yOgogICAgICAgICAgICByZXN1bHRzLmFwcGVuZCgoVHJ1ZSwgKGF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpKSkKCiAgICByZXR1cm4gcmVzdWx0cwoKCmRlZiBfcGFyYWxsZWxfcnVuKAogICAgbl93b3JrZXJzOiBpbnQsCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgUnVuIG11bHRpcGxlIFZBRCB3b3JrZXJzIHdpdGggbXVsdGlwcm9jZXNzaW5nIHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcKICAgIHRoZSBnaXZlbiB0YXNrIGNyZWF0b3IuCgogICAgOnBhcmFtIG5fd29ya2VyczogICAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZS4KICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgIFRoZSBhdWRpbyBmaWxlcyB0byB1c2UuCiAgICA6cGFyYW0gZGVzY3JpcHRpb246ICAgICBUaGUgZGVzY3JpcHRpb24gdG8gdXNlIGZvciB0aGUgcHJvZ3Jlc3MgYmFyLgogICAgOnBhcmFtIHZhZF9pbml0X2t3YXJnczogVGhlIFZBRCBpbml0aWFsaXphdGlvbiBrZXl3b3JkIGFyZ3VtZW50cy4KICAgIDpwYXJhbSB0YXNrX2NyZWF0b3I6ICAgIFRoZSB0YXNrIGNyZWF0b3IgdG8gdXNlIHRvIGNyZWF0ZSB0aGUgdGFza3MuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICBWZXJib3NpdHkuCgogICAgOnJldHVybnM6IFRoZSBjb2xsZWN0ZWQgcmVzdWx0cy4KICAgICIiIgogICAgIyBMb2FkIHRoZSBWQUQgKGRvd25sb2FkIG9uY2UsIGFuZCBpdCB3aWxsIGJlIGxvYWRlZCB0aGVuIHBlciBwcm9jZXNzIGxhdGVyIG9uKToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTG9hZGluZyB0aGUgVkFEIG1vZGVsLiIpCiAgICB2YWQgPSBWb2ljZUFjdGl2aXR5RGV0ZWN0b3IoKip2YWRfaW5pdF9rd2FyZ3MpCiAgICB2YWQubG9hZCgpCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiVkFEIG1vZGVsIGxvYWRlZC4iKQoKICAgICMgQ2hlY2sgdGhlIG51bWJlciBvZiB3b3JrZXJzOgogICAgaWYgbl93b3JrZXJzID4gbGVuKGF1ZGlvX2ZpbGVzKToKICAgICAgICBfTE9HR0VSLndhcm5pbmcoCiAgICAgICAgICAgIGYiVGhlIG51bWJlciBvZiB3b3JrZXJzICh7bl93b3JrZXJzfSkgaXMgbGFyZ2VyIHRoYW4gdGhlIG51bWJlciBvZiBhdWRpbyBmaWxlcyAoe2xlbihhdWRpb19maWxlcyl9KS4gIgogICAgICAgICAgICBmIlNldHRpbmcgdGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHtsZW4oYXVkaW9fZmlsZXMpfS4iCiAgICAgICAgKQogICAgICAgIG5fd29ya2VycyA9IGxlbihhdWRpb19maWxlcykKCiAgICAjIEluaXRpYWxpemUgdGhlIG11bHRpcHJvY2Vzc2luZyBxdWV1ZXM6CiAgICB0YXNrc19xdWV1ZSA9IFF1ZXVlKCkKICAgIHJlc3VsdHNfcXVldWUgPSBRdWV1ZSgpCgogICAgIyBJbml0aWFsaXplIHRoZSBtdWx0aXByb2Nlc3NpbmcgcHJvY2Vzc2VzOgogICAgdGFza19jb21wbGV0aW9uX3Byb2Nlc3NlcyA9IFsKICAgICAgICBQcm9jZXNzKAogICAgICAgICAgICB0YXJnZXQ9X211bHRpcHJvY2Vzc2luZ19jb21wbGV0ZV90YXNrcywKICAgICAgICAgICAga3dhcmdzPXsKICAgICAgICAgICAgICAgICJ2YWRfaW5pdF9rd2FyZ3MiOiB2YWRfaW5pdF9rd2FyZ3MsCiAgICAgICAgICAgICAgICAidGFza3NfcXVldWUiOiB0YXNrc19xdWV1ZSwKICAgICAgICAgICAgICAgICJyZXN1bHRzX3F1ZXVlIjogcmVzdWx0c19xdWV1ZSwKICAgICAgICAgICAgfSwKICAgICAgICApCiAgICAgICAgZm9yIF8gaW4gcmFuZ2Uobl93b3JrZXJzKQogICAgXQoKICAgICMgU3RhcnQgdGhlIG11bHRpcHJvY2Vzc2luZyBwcm9jZXNzZXM6CiAgICBmb3IgcCBpbiB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzOgogICAgICAgIHAuc3RhcnQoKQoKICAgICMgUHV0IHRoZSB0YXNrcyBpbiB0aGUgcXVldWU6CiAgICBmb3IgYXVkaW9fZmlsZSBpbiBhdWRpb19maWxlczoKICAgICAgICB0YXNrc19xdWV1ZS5wdXQodGFza19jcmVhdG9yLmNyZWF0ZV90YXNrKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSkudG9fdHVwbGUoKSkKCiAgICAjIFB1dCB0aGUgc3RvcCBtYXJrcyBpbiB0aGUgcXVldWU6CiAgICBmb3IgXyBpbiByYW5nZShuX3dvcmtlcnMpOgogICAgICAgIHRhc2tzX3F1ZXVlLnB1dChfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSykKCiAgICAjIENvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIHN0b3BfbWFya3NfY291bnRlciA9IDAKICAgIHdpdGggdHFkbSgKICAgICAgICBkZXNjPWRlc2NyaXB0aW9uLAogICAgICAgIHVuaXQ9ImZpbGUiLAogICAgICAgIHRvdGFsPWxlbihhdWRpb19maWxlcyksCiAgICAgICAgZGlzYWJsZT1ub3QgdmVyYm9zZSwKICAgICkgYXMgcHJvZ3Jlc3NiYXI6CiAgICAgICAgd2hpbGUgVHJ1ZToKICAgICAgICAgICAgIyBHZXQgYSByZXN1bHQgZnJvbSB0aGUgcXVldWU6CiAgICAgICAgICAgIHJlc3VsdDogVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBsaXN0XV0gPSByZXN1bHRzX3F1ZXVlLmdldCgpCiAgICAgICAgICAgIGlmIHJlc3VsdCA9PSBfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSzoKICAgICAgICAgICAgICAgIHN0b3BfbWFya3NfY291bnRlciArPSAxCiAgICAgICAgICAgICAgICBpZiBzdG9wX21hcmtzX2NvdW50ZXIgPT0gbl93b3JrZXJzOgogICAgICAgICAgICAgICAgICAgIGJyZWFrCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAjIENvbGxlY3QgdGhlIHJlc3VsdDoKICAgICAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKHJlc3VsdCkKICAgICAgICAgICAgICAgIHByb2dyZXNzYmFyLnVwZGF0ZSgxKQoKICAgICMgV2FpdCBmb3IgdGhlIHByb2Nlc3NlcyB0byBmaW5pc2g6CiAgICBmb3IgcCBpbiB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzOgogICAgICAgIHAuam9pbigpCgogICAgcmV0dXJuIHJlc3VsdHMKCgpkZWYgX3Byb2Nlc3NfcmVzdWx0cygKICAgIHJlc3VsdHM6IExpc3RbVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBsaXN0XV1dLCB2ZXJib3NlOiBib29sCikgLT4gVHVwbGVbZGljdCwgZGljdF06CiAgICAiIiIKICAgIFByb2Nlc3MgdGhlIHJlc3VsdHMgb2YgdGhlIHRhc2tzLgoKICAgIDpwYXJhbSByZXN1bHRzOiBUaGUgcmVzdWx0cyB0byBwcm9jZXNzLgogICAgOnBhcmFtIHZlcmJvc2U6IFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIHByb2Nlc3NlZCByZXN1bHRzIGFzIGEgdHVwbGUgb2Ygc3VjY2Vzc2VzIGFuZCBlcnJvcnMuCiAgICAiIiIKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJTdW1tYXJpemluZyB0aGUgcmVzdWx0cy4iKQogICAgc3VjY2Vzc2VzID0ge30KICAgIGVycm9ycyA9IHt9CiAgICBmb3IgaXNfZXJyb3IsIHJlc3VsdCBpbiByZXN1bHRzOgogICAgICAgIGlmIGlzX2Vycm9yOgogICAgICAgICAgICBlcnJvcnNbcmVzdWx0WzBdXSA9IHJlc3VsdFsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHN1Y2Nlc3Nlc1tyZXN1bHRbMF1dID0gcmVzdWx0WzFdCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkRvbmUgKHtsZW4oc3VjY2Vzc2VzKX0ve2xlbihzdWNjZXNzZXMpICsgbGVuKGVycm9ycyl9KVxuIikKCiAgICByZXR1cm4gc3VjY2Vzc2VzLCBlcnJvcnMK base_image: mlrun/mlrun commands: [] code_origin: '' @@ -48,7 +48,7 @@ spec: the VAD. outputs: - default: '' - lineno: 86 + lineno: 94 get_result: name: get_result doc: Get the result of the task. A tuple of the audio file name and the result. @@ -58,6 +58,16 @@ spec: - doc: The result of the task. default: '' lineno: 61 + to_tuple: + name: to_tuple + doc: Convert the task to a tuple to reconstruct it later (used for multiprocessing + to pass in queue). + parameters: + - name: self + outputs: + - doc: The converted task. + default: '' + lineno: 116 create_task: name: create_task doc: Create a task with the given audio file. @@ -68,16 +78,35 @@ spec: doc: The audio file to assign to the task. outputs: - doc: The created task. + type: BaseTask + default: '' + lineno: 146 + from_tuple: + name: from_tuple + doc: Create a task from a tuple of the audio file name and the task kwargs. + parameters: + - name: cls + - name: task_tuple + type: Tuple[str, dict] + doc: The task tuple to create the task from. + outputs: + - doc: The created task. + type: BaseTask default: '' - lineno: 123 + lineno: 157 load: name: load doc: Load the VAD model. parameters: - name: self + - name: force_reload + type: bool + doc: Whether to force reload the model even if it was already loaded. Default + is True. + default: true outputs: - default: '' - lineno: 199 + lineno: 234 detect_voice: name: detect_voice doc: "Perform voice activity detection on given audio files using the silero\ @@ -159,7 +188,7 @@ spec: default: false outputs: - default: '' - lineno: 347 + lineno: 393 diarize: name: diarize doc: "Perform speech diarization on given audio files using the silero VAD model\ @@ -237,7 +266,7 @@ spec: default: false outputs: - default: '' - lineno: 471 + lineno: 517 description: Silero VAD (Voice Activity Detection) functions. default_handler: detect_voice disable_auto_mount: false diff --git a/silero_vad/item.yaml b/silero_vad/item.yaml index 4b6c74fc5..6f85a4c7d 100644 --- a/silero_vad/item.yaml +++ b/silero_vad/item.yaml @@ -27,4 +27,4 @@ spec: - tqdm - onnxruntime url: '' -version: 1.0.0 +version: 1.1.0 diff --git a/silero_vad/silero_vad.py b/silero_vad/silero_vad.py index 08ff585fa..a477d4ecf 100644 --- a/silero_vad/silero_vad.py +++ b/silero_vad/silero_vad.py @@ -765,7 +765,7 @@ def _parallel_run( n_workers = len(audio_files) # Initialize the multiprocessing queues: - tasks_queue = Queue(maxsize=n_workers * 2) + tasks_queue = Queue() results_queue = Queue() # Initialize the multiprocessing processes: @@ -785,39 +785,38 @@ def _parallel_run( for p in task_completion_processes: p.start() - # Put the tasks in the queue (the progress bar is not accurate as it is updating by the queue which has a max size - # of 2*n_workers so the progress bar won't be too off - better than nothing): - for audio_file in tqdm( - audio_files, - desc=description, - unit="file", - total=len(audio_files), - disable=not verbose, - ): - # Put the task in the queue: + # Put the tasks in the queue: + for audio_file in audio_files: tasks_queue.put(task_creator.create_task(audio_file=audio_file).to_tuple()) # Put the stop marks in the queue: for _ in range(n_workers): tasks_queue.put(_MULTIPROCESSING_STOP_MARK) - # Wait for the processes to finish: - for p in task_completion_processes: - p.join() - # Collect the results: results = [] stop_marks_counter = 0 - while True: - # Get a result from the queue: - result: Tuple[bool, Tuple[str, list]] = results_queue.get() - if result == _MULTIPROCESSING_STOP_MARK: - stop_marks_counter += 1 - if stop_marks_counter == n_workers: - break - else: - # Collect the result: - results.append(result) + with tqdm( + desc=description, + unit="file", + total=len(audio_files), + disable=not verbose, + ) as progressbar: + while True: + # Get a result from the queue: + result: Tuple[bool, Tuple[str, list]] = results_queue.get() + if result == _MULTIPROCESSING_STOP_MARK: + stop_marks_counter += 1 + if stop_marks_counter == n_workers: + break + else: + # Collect the result: + results.append(result) + progressbar.update(1) + + # Wait for the processes to finish: + for p in task_completion_processes: + p.join() return results