diff --git a/bert_embeddings/function.yaml b/bert_embeddings/function.yaml index 4a3fcf54f..15319c160 100644 --- a/bert_embeddings/function.yaml +++ b/bert_embeddings/function.yaml @@ -2,13 +2,15 @@ kind: remote metadata: name: bert-embeddings tag: '' - hash: 57a2ce8e0da1f6e813a8649e9ea6fcbb69a1ce5f + hash: ecf6647fe4716e0df54ce50278b735034536a568 project: '' labels: framework: pytorch categories: + - huggingface - machine-learning - data-preparation + - pytorch spec: command: '' args: [] @@ -16,15 +18,17 @@ spec: build: functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGpzb24KaW1wb3J0IHBpY2tsZQoKaW1wb3J0IHRvcmNoCmZyb20gdHJhbnNmb3JtZXJzIGltcG9ydCBCZXJ0TW9kZWwsIEJlcnRUb2tlbml6ZXIKCgpkZWYgaW5pdF9jb250ZXh0KGNvbnRleHQpOgogICAgdG9rZW5pemVyID0gQmVydFRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoImJlcnQtYmFzZS11bmNhc2VkIikKICAgIG1vZGVsID0gQmVydE1vZGVsLmZyb21fcHJldHJhaW5lZCgiYmVydC1iYXNlLXVuY2FzZWQiKQogICAgbW9kZWwuZXZhbCgpCgogICAgc2V0YXR0cihjb250ZXh0LnVzZXJfZGF0YSwgInRva2VuaXplciIsIHRva2VuaXplcikKICAgIHNldGF0dHIoY29udGV4dC51c2VyX2RhdGEsICJtb2RlbCIsIG1vZGVsKQoKCmRlZiBoYW5kbGVyKGNvbnRleHQsIGV2ZW50KToKICAgIGRvY3MgPSBqc29uLmxvYWRzKGV2ZW50LmJvZHkpCiAgICBkb2NzID0gW2RvYy5sb3dlcigpIGZvciBkb2MgaW4gZG9jc10KICAgIGRvY3MgPSBjb250ZXh0LnVzZXJfZGF0YS50b2tlbml6ZXIuYmF0Y2hfZW5jb2RlX3BsdXMoCiAgICAgICAgZG9jcywgcGFkX3RvX21heF9sZW5ndGg9VHJ1ZSwgcmV0dXJuX3RlbnNvcnM9InB0IgogICAgKQoKICAgIHdpdGggdG9yY2gubm9fZ3JhZCgpOgogICAgICAgIGVtYmVkZGluZ3MgPSBjb250ZXh0LnVzZXJfZGF0YS5tb2RlbCgqKmRvY3MpCiAgICBlbWJlZGRpbmdzID0gW2VtYmVkZGluZ3NbMF0ubnVtcHkoKSwgZW1iZWRkaW5nc1sxXS5udW1weSgpXQogICAgcmV0dXJuIHBpY2tsZS5kdW1wcyhlbWJlZGRpbmdzKQo= commands: [] - code_origin: http://github.com/aviaIguazio/functions.git#a1c9940e4c2420c88063768b4038e29b1f4e37a6:/Users/Avi_Asulin/PycharmProjects/mlrun/functions/bert_embeddings/bert_embeddings.py - origin_filename: /Users/Avi_Asulin/PycharmProjects/mlrun/functions/bert_embeddings/bert_embeddings.py + code_origin: '' + origin_filename: '' requirements: - torch description: Get BERT based embeddings for given text default_handler: '' disable_auto_mount: false clone_target_dir: '' - env: [] + env: + - name: MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK + value: enabled priority_class_name: '' preemption_mode: prevent min_replicas: 1 diff --git a/bert_embeddings/item.yaml b/bert_embeddings/item.yaml index f0eaed1c0..f96e54eae 100644 --- a/bert_embeddings/item.yaml +++ b/bert_embeddings/item.yaml @@ -1,7 +1,9 @@ apiVersion: v1 categories: +- huggingface - machine-learning - data-preparation +- pytorch description: Get BERT based embeddings for given text doc: '' example: bert_embeddings.ipynb @@ -23,4 +25,4 @@ spec: requirements: - torch url: '' -version: 1.2.0 +version: 1.3.0 diff --git a/hugging_face_classifier_trainer/function.yaml b/hugging_face_classifier_trainer/function.yaml index eb223b2bf..65f5aeb10 100644 --- a/hugging_face_classifier_trainer/function.yaml +++ b/hugging_face_classifier_trainer/function.yaml @@ -2,11 +2,13 @@ kind: job metadata: name: hugging-face-classifier-trainer tag: '' - hash: e8113e81f04c96fc9a8a94e717dea81ee3e05a18 + hash: f9d8aa4a2c66e24fa418bb163829adc3e2ada06c project: '' labels: author: davids categories: + - deep-learning + - huggingface - machine-learning - model-training spec: diff --git a/hugging_face_classifier_trainer/item.yaml b/hugging_face_classifier_trainer/item.yaml index 3c0877659..332902b3e 100755 --- a/hugging_face_classifier_trainer/item.yaml +++ b/hugging_face_classifier_trainer/item.yaml @@ -1,5 +1,7 @@ apiVersion: v1 categories: +- deep-learning +- huggingface - machine-learning - model-training description: Automatic train and optimize functions for HuggingFace framework @@ -28,4 +30,4 @@ spec: - datasets~=2.10.1 - scikit-learn~=1.0.2 url: '' -version: 0.2.0 +version: 0.3.0 diff --git a/hugging_face_serving/function.yaml b/hugging_face_serving/function.yaml index e1bb3b0ce..764fc1cfe 100644 --- a/hugging_face_serving/function.yaml +++ b/hugging_face_serving/function.yaml @@ -2,11 +2,13 @@ kind: serving metadata: name: hugging-face-serving tag: '' - hash: 39bfca7b639022fa03f5ca87f85f9e17fc837b70 + hash: 1a489a57da861f129eb26e933f34e58927e41195 project: '' labels: author: yonish categories: + - huggingface + - genai - model-serving - machine-learning spec: @@ -14,37 +16,28 @@ spec: args: [] image: mlrun/ml-models build: - commands: - - python -m pip install transformers==4.21.3 tensorflow==2.9.2 - code_origin: https://github.com/mlrun/functions.git#250244b2527c5ce8a82438b4340df34de6e19dc3:/Users/yonatanshelach/yoni/projects/functions/hugging_face_serving/hugging_face_serving.py - origin_filename: /Users/yonatanshelach/yoni/projects/functions/hugging_face_serving/hugging_face_serving.py + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCmZyb20gYWJjIGltcG9ydCBBQkMKZnJvbSBpbXBvcnRsaWIgaW1wb3J0IGltcG9ydF9tb2R1bGUKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QKCmZyb20gdHJhbnNmb3JtZXJzIGltcG9ydCBwaXBlbGluZQoKaW1wb3J0IG1scnVuLnNlcnZpbmcKClBBQ0tBR0VfTU9EVUxFID0gInRyYW5zZm9ybWVycyIKU0VSSUFMSVpBQkxFX1RZUEVTID0gW2RpY3QsIGxpc3QsIHR1cGxlLCBzdHIsIGludCwgZmxvYXRdCgoKY2xhc3MgSHVnZ2luZ0ZhY2VNb2RlbFNlcnZlcihtbHJ1bi5zZXJ2aW5nLlYyTW9kZWxTZXJ2ZXIsIEFCQyk6CiAgICAiIiIKICAgIEh1Z2dpbmcgRmFjZSBNb2RlbCBzZXJ2aW5nIGNsYXNzLCBpbmhlcml0aW5nIHRoZSBWMk1vZGVsU2VydmVyIGNsYXNzIGZvciBiZWluZyBpbml0aWFsaXplZCBhdXRvbWF0aWNhbGx5IGJ5IHRoZQogICAgbW9kZWwgc2VydmVyIGFuZCBiZSBhYmxlIHRvIHJ1biBsb2NhbGx5IGFzIHBhcnQgb2YgYSBudWNsaW8gc2VydmVybGVzcyBmdW5jdGlvbiwgb3IgYXMgcGFydCBvZiBhIHJlYWwtdGltZSBwaXBlbGluZS4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgICAgIG5hbWU6IHN0ciwKICAgICAgICB0YXNrOiBzdHIsCiAgICAgICAgbW9kZWxfcGF0aDogc3RyID0gTm9uZSwKICAgICAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG1vZGVsX2NsYXNzOiBzdHIgPSBOb25lLAogICAgICAgIHRva2VuaXplcl9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIHRva2VuaXplcl9jbGFzczogc3RyID0gTm9uZSwKICAgICAgICBmcmFtZXdvcms6IHN0ciA9IE5vbmUsCiAgICAgICAgKipjbGFzc19hcmdzLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIGEgc2VydmluZyBjbGFzcyBmb3IgYSBIdWdnaW5nIGZhY2UgbW9kZWwuCgogICAgICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIFRoZSBtbHJ1biBjb250ZXh0IHRvIHdvcmsgd2l0aAogICAgICAgIDpwYXJhbSBuYW1lOiAgICAgICAgICAgIFRoZSBuYW1lIG9mIHRoaXMgc2VydmVyIHRvIGJlIGluaXRpYWxpemVkCiAgICAgICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgTm90IGluIHVzZS4gV2hlbiBhZGRpbmcgYSBtb2RlbCBwYXNzIGFueSBzdHJpbmcgdmFsdWUKICAgICAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgICBUaGUgbW9kZWwncyBuYW1lIGluIHRoZSBIdWdnaW5nIEZhY2UgaHViCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZS5nLiwgYG5scHRvd24vYmVydC1iYXNlLW11bHRpbGluZ3VhbC11bmNhc2VkLXNlbnRpbWVudGAKICAgICAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICBUaGUgbW9kZWwncyBjbGFzcyB0eXBlIG9iamVjdCB3aGljaCBjYW4gYmUgcGFzc2VkIGFzIHRoZSBjbGFzcydzIG5hbWUgKHN0cmluZykuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTXVzdCBiZSBwcm92aWRlZCBhbmQgdG8gYmUgbWF0Y2hlZCB3aXRoIGBtb2RlbF9uYW1lYC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlLmcuLCBgQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbmAKICAgICAgICA6cGFyYW0gdG9rZW5pemVyX25hbWU6ICBUaGUgdG9rZW5pemVyJ3MgbmFtZSBpbiB0aGUgSHVnZ2luZyBGYWNlIGh1YgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGUuZy4sIGBubHB0b3duL2JlcnQtYmFzZS1tdWx0aWxpbmd1YWwtdW5jYXNlZC1zZW50aW1lbnRgCiAgICAgICAgOnBhcmFtIHRva2VuaXplcl9jbGFzczogVGhlIG1vZGVsJ3MgY2xhc3MgdHlwZSBvYmplY3Qgd2hpY2ggY2FuIGJlIHBhc3NlZCBhcyB0aGUgY2xhc3MncyBuYW1lIChzdHJpbmcpLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE11c3QgYmUgcHJvdmlkZWQgYW5kIHRvIGJlIG1hdGNoZWQgd2l0aCBgbW9kZWxfbmFtZWAuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZS5nLiwgYEF1dG9Ub2tlbml6ZXJgCiAgICAgICAgOnBhcmFtIGZyYW1ld29yazogICAgICAgVGhlIGZyYW1ld29yayB0byB1c2UsIGVpdGhlciBgInB0ImAgZm9yIFB5VG9yY2ggb3IgYCJ0ZiJgIGZvciBUZW5zb3JGbG93LiBUaGUgc3BlY2lmaWVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnJhbWV3b3JrIG11c3QgYmUgaW5zdGFsbGVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIG5vIGZyYW1ld29yayBpcyBzcGVjaWZpZWQsIHdpbGwgZGVmYXVsdCB0byB0aGUgb25lIGN1cnJlbnRseSBpbnN0YWxsZWQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgbm8gZnJhbWV3b3JrIGlzIHNwZWNpZmllZCBhbmQgYm90aCBmcmFtZXdvcmtzIGFyZSBpbnN0YWxsZWQsIHdpbGwgZGVmYXVsdCB0byB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmcmFtZXdvcmsgb2YgdGhlIGBtb2RlbGAsIG9yIHRvIFB5VG9yY2ggaWYgbm8gbW9kZWwgaXMgcHJvdmlkZWQuCiAgICAgICAgOnBhcmFtIGNsYXNzX2FyZ3M6ICAgICAgLQogICAgICAgICIiIgogICAgICAgIHN1cGVyKEh1Z2dpbmdGYWNlTW9kZWxTZXJ2ZXIsIHNlbGYpLl9faW5pdF9fKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIG5hbWU9bmFtZSwKICAgICAgICAgICAgbW9kZWxfcGF0aD1tb2RlbF9wYXRoLAogICAgICAgICAgICAqKmNsYXNzX2FyZ3MsCiAgICAgICAgKQogICAgICAgIHNlbGYudGFzayA9IHRhc2sKICAgICAgICBzZWxmLm1vZGVsID0gTm9uZQogICAgICAgIHNlbGYudG9rZW5pemVyID0gTm9uZQogICAgICAgIHNlbGYubW9kZWxfbmFtZSA9IG1vZGVsX25hbWUKICAgICAgICBzZWxmLnRva2VuaXplcl9uYW1lID0gdG9rZW5pemVyX25hbWUKICAgICAgICBzZWxmLm1vZGVsX2NsYXNzID0gbW9kZWxfY2xhc3MKICAgICAgICBzZWxmLnRva2VuaXplcl9jbGFzcyA9IHRva2VuaXplcl9jbGFzcwogICAgICAgIHNlbGYuZnJhbWV3b3JrID0gZnJhbWV3b3JrCiAgICAgICAgc2VsZi5waXBlID0gTm9uZQoKICAgIGRlZiBsb2FkKHNlbGYpOgogICAgICAgICIiImxvYWQgYW5kIGluaXRpYWxpemUgdGhlIG1vZGVsIGFuZC9vciBvdGhlciBlbGVtZW50cyIiIgogICAgICAgIGlmIHNlbGYubW9kZWxfY2xhc3M6CiAgICAgICAgICAgIG1vZGVsX29iamVjdCA9IGdldGF0dHIoaW1wb3J0X21vZHVsZShQQUNLQUdFX01PRFVMRSksIHNlbGYubW9kZWxfY2xhc3MpCiAgICAgICAgICAgIHNlbGYubW9kZWwgPSBtb2RlbF9vYmplY3QuZnJvbV9wcmV0cmFpbmVkKHNlbGYubW9kZWxfbmFtZSkKICAgICAgICBpZiBzZWxmLnRva2VuaXplcl9jbGFzczoKICAgICAgICAgICAgdG9rZW5pemVyX29iamVjdCA9IGdldGF0dHIoCiAgICAgICAgICAgICAgICBpbXBvcnRfbW9kdWxlKFBBQ0tBR0VfTU9EVUxFKSwgc2VsZi50b2tlbml6ZXJfY2xhc3MKICAgICAgICAgICAgKQogICAgICAgICAgICBzZWxmLnRva2VuaXplciA9IHRva2VuaXplcl9vYmplY3QuZnJvbV9wcmV0cmFpbmVkKHNlbGYudG9rZW5pemVyX25hbWUpCiAgICAgICAgc2VsZi5waXBlID0gcGlwZWxpbmUoCiAgICAgICAgICAgIHRhc2s9c2VsZi50YXNrLAogICAgICAgICAgICBtb2RlbD1zZWxmLm1vZGVsIG9yIHNlbGYubW9kZWxfbmFtZSwKICAgICAgICAgICAgdG9rZW5pemVyPXNlbGYudG9rZW5pemVyLAogICAgICAgICAgICBmcmFtZXdvcms9c2VsZi5mcmFtZXdvcmssCiAgICAgICAgKQoKICAgIGRlZiBwcmVkaWN0KHNlbGYsIGJvZHk6IGRpY3QpIC0+IExpc3Q6CiAgICAgICAgIiIiR2VuZXJhdGUgbW9kZWwgcHJlZGljdGlvbnMgZnJvbSBzYW1wbGUuIiIiCiAgICAgICAgaWYgc2VsZi5waXBlIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIlBsZWFzZSB1c2UgYC5sb2FkKClgIikKICAgICAgICB0cnk6CiAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoYm9keVsiaW5wdXRzIl1bMF0sIGRpY3QpOgogICAgICAgICAgICAgICAgcmVzdWx0ID0gW3NlbGYucGlwZSgqKl9pbnB1dCkgZm9yIF9pbnB1dCBpbiBib2R5WyJpbnB1dHMiXV0KICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIHJlc3VsdCA9IHNlbGYucGlwZShib2R5WyJpbnB1dHMiXSkKICAgICAgICAgICAgIyByZXBsYWNlIGxpc3Qgb2YgbGlzdHMgb2YgZGljdHMgaW50byBhIGxpc3Qgb2YgZGljdHM6CiAgICAgICAgICAgIGlmIGFsbChpc2luc3RhbmNlKHJlcywgbGlzdCkgZm9yIHJlcyBpbiByZXN1bHQpOgogICAgICAgICAgICAgICAgbmV3X3Jlc3VsdCA9IFtyZXNbMF0gZm9yIHJlcyBpbiByZXN1bHRdCiAgICAgICAgICAgICAgICByZXN1bHQgPSBuZXdfcmVzdWx0CgogICAgICAgICAgICBub25fc2VyaWFsaXphYmxlX3R5cGVzID0gW10KICAgICAgICAgICAgZm9yIHJlcyBpbiByZXN1bHQ6CiAgICAgICAgICAgICAgICBmb3Iga2V5LCB2YWwgaW4gcmVzLml0ZW1zKCk6CiAgICAgICAgICAgICAgICAgICAgaWYgdHlwZSh2YWwpIG5vdCBpbiBTRVJJQUxJWkFCTEVfVFlQRVM6CiAgICAgICAgICAgICAgICAgICAgICAgIG5vbl9zZXJpYWxpemFibGVfdHlwZXMuYXBwZW5kKHN0cih0eXBlKHZhbCkpKQogICAgICAgICAgICAgICAgICAgICAgICByZXNba2V5XSA9IHN0cih2YWwpCiAgICAgICAgICAgIGlmIG5vbl9zZXJpYWxpemFibGVfdHlwZXM6CiAgICAgICAgICAgICAgICBzZWxmLmNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJOb24tc2VyaWFsaXphYmxlIHR5cGVzOiB7bm9uX3NlcmlhbGl6YWJsZV90eXBlc30gd2VyZSBjYXN0ZWQgdG8gc3RyaW5ncyIKICAgICAgICAgICAgICAgICkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiRmFpbGVkIHRvIHByZWRpY3QgJXMiICUgZSkKICAgICAgICByZXR1cm4gcmVzdWx0Cgpmcm9tIG1scnVuLnJ1bnRpbWVzIGltcG9ydCBudWNsaW9faW5pdF9ob29rCmRlZiBpbml0X2NvbnRleHQoY29udGV4dCk6CiAgICBudWNsaW9faW5pdF9ob29rKGNvbnRleHQsIGdsb2JhbHMoKSwgJ3NlcnZpbmdfdjInKQoKZGVmIGhhbmRsZXIoY29udGV4dCwgZXZlbnQpOgogICAgcmV0dXJuIGNvbnRleHQubWxydW5faGFuZGxlcihjb250ZXh0LCBldmVudCkK + commands: [] + code_origin: '' + origin_filename: '' + requirements: + - transformers==4.21.3 + - tensorflow==2.9.2 description: Generic Hugging Face model server. - default_handler: handler + default_handler: '' disable_auto_mount: false - env: [] + clone_target_dir: '' + env: + - name: MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK + value: enabled priority_class_name: '' preemption_mode: prevent min_replicas: 1 max_replicas: 4 - base_spec: - apiVersion: nuclio.io/v1 - kind: Function - metadata: - name: hugging-face-serving - labels: {} - annotations: - nuclio.io/generated_by: function generated from /Users/yonatanshelach/yoni/projects/functions/hugging_face_serving/hugging_face_serving.py - spec: - runtime: python - handler: hugging_face_serving:handler - env: [] - volumes: [] - build: - commands: [] - noBaseImagesPull: true - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCmZyb20gYWJjIGltcG9ydCBBQkMKZnJvbSBpbXBvcnRsaWIgaW1wb3J0IGltcG9ydF9tb2R1bGUKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QKCmZyb20gdHJhbnNmb3JtZXJzIGltcG9ydCBwaXBlbGluZQoKaW1wb3J0IG1scnVuLnNlcnZpbmcKClBBQ0tBR0VfTU9EVUxFID0gInRyYW5zZm9ybWVycyIKU0VSSUFMSVpBQkxFX1RZUEVTID0gW2RpY3QsIGxpc3QsIHR1cGxlLCBzdHIsIGludCwgZmxvYXRdCgoKY2xhc3MgSHVnZ2luZ0ZhY2VNb2RlbFNlcnZlcihtbHJ1bi5zZXJ2aW5nLlYyTW9kZWxTZXJ2ZXIsIEFCQyk6CiAgICAiIiIKICAgIEh1Z2dpbmcgRmFjZSBNb2RlbCBzZXJ2aW5nIGNsYXNzLCBpbmhlcml0aW5nIHRoZSBWMk1vZGVsU2VydmVyIGNsYXNzIGZvciBiZWluZyBpbml0aWFsaXplZCBhdXRvbWF0aWNhbGx5IGJ5IHRoZQogICAgbW9kZWwgc2VydmVyIGFuZCBiZSBhYmxlIHRvIHJ1biBsb2NhbGx5IGFzIHBhcnQgb2YgYSBudWNsaW8gc2VydmVybGVzcyBmdW5jdGlvbiwgb3IgYXMgcGFydCBvZiBhIHJlYWwtdGltZSBwaXBlbGluZS4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgICAgIG5hbWU6IHN0ciwKICAgICAgICB0YXNrOiBzdHIsCiAgICAgICAgbW9kZWxfcGF0aDogc3RyID0gTm9uZSwKICAgICAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG1vZGVsX2NsYXNzOiBzdHIgPSBOb25lLAogICAgICAgIHRva2VuaXplcl9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIHRva2VuaXplcl9jbGFzczogc3RyID0gTm9uZSwKICAgICAgICBmcmFtZXdvcms6IHN0ciA9IE5vbmUsCiAgICAgICAgKipjbGFzc19hcmdzLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIGEgc2VydmluZyBjbGFzcyBmb3IgYSBIdWdnaW5nIGZhY2UgbW9kZWwuCgogICAgICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIFRoZSBtbHJ1biBjb250ZXh0IHRvIHdvcmsgd2l0aAogICAgICAgIDpwYXJhbSBuYW1lOiAgICAgICAgICAgIFRoZSBuYW1lIG9mIHRoaXMgc2VydmVyIHRvIGJlIGluaXRpYWxpemVkCiAgICAgICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgTm90IGluIHVzZS4gV2hlbiBhZGRpbmcgYSBtb2RlbCBwYXNzIGFueSBzdHJpbmcgdmFsdWUKICAgICAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgICBUaGUgbW9kZWwncyBuYW1lIGluIHRoZSBIdWdnaW5nIEZhY2UgaHViCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZS5nLiwgYG5scHRvd24vYmVydC1iYXNlLW11bHRpbGluZ3VhbC11bmNhc2VkLXNlbnRpbWVudGAKICAgICAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICBUaGUgbW9kZWwncyBjbGFzcyB0eXBlIG9iamVjdCB3aGljaCBjYW4gYmUgcGFzc2VkIGFzIHRoZSBjbGFzcydzIG5hbWUgKHN0cmluZykuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTXVzdCBiZSBwcm92aWRlZCBhbmQgdG8gYmUgbWF0Y2hlZCB3aXRoIGBtb2RlbF9uYW1lYC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlLmcuLCBgQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbmAKICAgICAgICA6cGFyYW0gdG9rZW5pemVyX25hbWU6ICBUaGUgdG9rZW5pemVyJ3MgbmFtZSBpbiB0aGUgSHVnZ2luZyBGYWNlIGh1YgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGUuZy4sIGBubHB0b3duL2JlcnQtYmFzZS1tdWx0aWxpbmd1YWwtdW5jYXNlZC1zZW50aW1lbnRgCiAgICAgICAgOnBhcmFtIHRva2VuaXplcl9jbGFzczogVGhlIG1vZGVsJ3MgY2xhc3MgdHlwZSBvYmplY3Qgd2hpY2ggY2FuIGJlIHBhc3NlZCBhcyB0aGUgY2xhc3MncyBuYW1lIChzdHJpbmcpLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE11c3QgYmUgcHJvdmlkZWQgYW5kIHRvIGJlIG1hdGNoZWQgd2l0aCBgbW9kZWxfbmFtZWAuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZS5nLiwgYEF1dG9Ub2tlbml6ZXJgCiAgICAgICAgOnBhcmFtIGZyYW1ld29yazogICAgICAgVGhlIGZyYW1ld29yayB0byB1c2UsIGVpdGhlciBgInB0ImAgZm9yIFB5VG9yY2ggb3IgYCJ0ZiJgIGZvciBUZW5zb3JGbG93LiBUaGUgc3BlY2lmaWVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnJhbWV3b3JrIG11c3QgYmUgaW5zdGFsbGVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIG5vIGZyYW1ld29yayBpcyBzcGVjaWZpZWQsIHdpbGwgZGVmYXVsdCB0byB0aGUgb25lIGN1cnJlbnRseSBpbnN0YWxsZWQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgbm8gZnJhbWV3b3JrIGlzIHNwZWNpZmllZCBhbmQgYm90aCBmcmFtZXdvcmtzIGFyZSBpbnN0YWxsZWQsIHdpbGwgZGVmYXVsdCB0byB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmcmFtZXdvcmsgb2YgdGhlIGBtb2RlbGAsIG9yIHRvIFB5VG9yY2ggaWYgbm8gbW9kZWwgaXMgcHJvdmlkZWQuCiAgICAgICAgOnBhcmFtIGNsYXNzX2FyZ3M6ICAgICAgLQogICAgICAgICIiIgogICAgICAgIHN1cGVyKEh1Z2dpbmdGYWNlTW9kZWxTZXJ2ZXIsIHNlbGYpLl9faW5pdF9fKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIG5hbWU9bmFtZSwKICAgICAgICAgICAgbW9kZWxfcGF0aD1tb2RlbF9wYXRoLAogICAgICAgICAgICAqKmNsYXNzX2FyZ3MsCiAgICAgICAgKQogICAgICAgIHNlbGYudGFzayA9IHRhc2sKICAgICAgICBzZWxmLm1vZGVsID0gTm9uZQogICAgICAgIHNlbGYudG9rZW5pemVyID0gTm9uZQogICAgICAgIHNlbGYubW9kZWxfbmFtZSA9IG1vZGVsX25hbWUKICAgICAgICBzZWxmLnRva2VuaXplcl9uYW1lID0gdG9rZW5pemVyX25hbWUKICAgICAgICBzZWxmLm1vZGVsX2NsYXNzID0gbW9kZWxfY2xhc3MKICAgICAgICBzZWxmLnRva2VuaXplcl9jbGFzcyA9IHRva2VuaXplcl9jbGFzcwogICAgICAgIHNlbGYuZnJhbWV3b3JrID0gZnJhbWV3b3JrCiAgICAgICAgc2VsZi5waXBlID0gTm9uZQoKICAgIGRlZiBsb2FkKHNlbGYpOgogICAgICAgICIiImxvYWQgYW5kIGluaXRpYWxpemUgdGhlIG1vZGVsIGFuZC9vciBvdGhlciBlbGVtZW50cyIiIgogICAgICAgIGlmIHNlbGYubW9kZWxfY2xhc3M6CiAgICAgICAgICAgIG1vZGVsX29iamVjdCA9IGdldGF0dHIoaW1wb3J0X21vZHVsZShQQUNLQUdFX01PRFVMRSksIHNlbGYubW9kZWxfY2xhc3MpCiAgICAgICAgICAgIHNlbGYubW9kZWwgPSBtb2RlbF9vYmplY3QuZnJvbV9wcmV0cmFpbmVkKHNlbGYubW9kZWxfbmFtZSkKICAgICAgICBpZiBzZWxmLnRva2VuaXplcl9jbGFzczoKICAgICAgICAgICAgdG9rZW5pemVyX29iamVjdCA9IGdldGF0dHIoCiAgICAgICAgICAgICAgICBpbXBvcnRfbW9kdWxlKFBBQ0tBR0VfTU9EVUxFKSwgc2VsZi50b2tlbml6ZXJfY2xhc3MKICAgICAgICAgICAgKQogICAgICAgICAgICBzZWxmLnRva2VuaXplciA9IHRva2VuaXplcl9vYmplY3QuZnJvbV9wcmV0cmFpbmVkKHNlbGYudG9rZW5pemVyX25hbWUpCiAgICAgICAgc2VsZi5waXBlID0gcGlwZWxpbmUoCiAgICAgICAgICAgIHRhc2s9c2VsZi50YXNrLAogICAgICAgICAgICBtb2RlbD1zZWxmLm1vZGVsIG9yIHNlbGYubW9kZWxfbmFtZSwKICAgICAgICAgICAgdG9rZW5pemVyPXNlbGYudG9rZW5pemVyLAogICAgICAgICAgICBmcmFtZXdvcms9c2VsZi5mcmFtZXdvcmssCiAgICAgICAgKQoKICAgIGRlZiBwcmVkaWN0KHNlbGYsIGJvZHk6IGRpY3QpIC0+IExpc3Q6CiAgICAgICAgIiIiR2VuZXJhdGUgbW9kZWwgcHJlZGljdGlvbnMgZnJvbSBzYW1wbGUuIiIiCiAgICAgICAgaWYgc2VsZi5waXBlIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIlBsZWFzZSB1c2UgYC5sb2FkKClgIikKICAgICAgICB0cnk6CiAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoYm9keVsiaW5wdXRzIl1bMF0sIGRpY3QpOgogICAgICAgICAgICAgICAgcmVzdWx0ID0gW3NlbGYucGlwZSgqKl9pbnB1dCkgZm9yIF9pbnB1dCBpbiBib2R5WyJpbnB1dHMiXV0KICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIHJlc3VsdCA9IHNlbGYucGlwZShib2R5WyJpbnB1dHMiXSkKICAgICAgICAgICAgIyByZXBsYWNlIGxpc3Qgb2YgbGlzdHMgb2YgZGljdHMgaW50byBhIGxpc3Qgb2YgZGljdHM6CiAgICAgICAgICAgIGlmIGFsbChpc2luc3RhbmNlKHJlcywgbGlzdCkgZm9yIHJlcyBpbiByZXN1bHQpOgogICAgICAgICAgICAgICAgbmV3X3Jlc3VsdCA9IFtyZXNbMF0gZm9yIHJlcyBpbiByZXN1bHRdCiAgICAgICAgICAgICAgICByZXN1bHQgPSBuZXdfcmVzdWx0CgogICAgICAgICAgICBub25fc2VyaWFsaXphYmxlX3R5cGVzID0gW10KICAgICAgICAgICAgZm9yIHJlcyBpbiByZXN1bHQ6CiAgICAgICAgICAgICAgICBmb3Iga2V5LCB2YWwgaW4gcmVzLml0ZW1zKCk6CiAgICAgICAgICAgICAgICAgICAgaWYgdHlwZSh2YWwpIG5vdCBpbiBTRVJJQUxJWkFCTEVfVFlQRVM6CiAgICAgICAgICAgICAgICAgICAgICAgIG5vbl9zZXJpYWxpemFibGVfdHlwZXMuYXBwZW5kKHN0cih0eXBlKHZhbCkpKQogICAgICAgICAgICAgICAgICAgICAgICByZXNba2V5XSA9IHN0cih2YWwpCiAgICAgICAgICAgIGlmIG5vbl9zZXJpYWxpemFibGVfdHlwZXM6CiAgICAgICAgICAgICAgICBzZWxmLmNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJOb24tc2VyaWFsaXphYmxlIHR5cGVzOiB7bm9uX3NlcmlhbGl6YWJsZV90eXBlc30gd2VyZSBjYXN0ZWQgdG8gc3RyaW5ncyIKICAgICAgICAgICAgICAgICkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiRmFpbGVkIHRvIHByZWRpY3QgJXMiICUgZSkKICAgICAgICByZXR1cm4gcmVzdWx0Cgpmcm9tIG1scnVuLnJ1bnRpbWVzIGltcG9ydCBudWNsaW9faW5pdF9ob29rCmRlZiBpbml0X2NvbnRleHQoY29udGV4dCk6CiAgICBudWNsaW9faW5pdF9ob29rKGNvbnRleHQsIGdsb2JhbHMoKSwgJ3NlcnZpbmdfdjInKQoKZGVmIGhhbmRsZXIoY29udGV4dCwgZXZlbnQpOgogICAgcmV0dXJuIGNvbnRleHQubWxydW5faGFuZGxlcihjb250ZXh0LCBldmVudCkK source: '' function_kind: serving_v2 + function_handler: hugging_face_serving:handler + base_image_pull: false default_class: HuggingFaceModelServer secret_sources: [] affinity: null diff --git a/hugging_face_serving/item.yaml b/hugging_face_serving/item.yaml index f7fa92637..d1f78769d 100644 --- a/hugging_face_serving/item.yaml +++ b/hugging_face_serving/item.yaml @@ -1,5 +1,7 @@ apiVersion: v1 categories: +- huggingface +- genai - model-serving - machine-learning description: Generic Hugging Face model server. @@ -26,4 +28,5 @@ spec: - transformers==4.21.3 - tensorflow==2.9.2 url: '' -version: 1.0.0 +version: 1.1.0 +test_valid: false \ No newline at end of file diff --git a/huggingface_auto_trainer/function.yaml b/huggingface_auto_trainer/function.yaml index eff09b4cf..702a84016 100644 --- a/huggingface_auto_trainer/function.yaml +++ b/huggingface_auto_trainer/function.yaml @@ -2,11 +2,13 @@ kind: job metadata: name: huggingface-auto-trainer tag: '' - hash: 4459f0b675c36a20c8f542126a96b98b0ac82271 + hash: 55c9aa4a822780f7388819ccf633dfe26b31f02e project: '' labels: author: Zeevr categories: + - huggingface + - genai - machine-learning - model-training spec: @@ -16,8 +18,8 @@ spec: build: functionSourceCode: aW1wb3J0IGltcG9ydGxpYgppbXBvcnQgb3MKaW1wb3J0IHNodXRpbAppbXBvcnQgdGVtcGZpbGUKaW1wb3J0IHppcGZpbGUKZnJvbSBhYmMgaW1wb3J0IEFCQwpmcm9tIHR5cGluZyBpbXBvcnQgRGljdCwgTGlzdCwgVHVwbGUsIFVuaW9uCgppbXBvcnQgbWxydW4KaW1wb3J0IG51bXB5IGFzIG5wCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHBlZnQKaW1wb3J0IHRvcmNoCmltcG9ydCB0cmFuc2Zvcm1lcnMKZnJvbSBkYXRhc2V0cyBpbXBvcnQgRGF0YXNldCwgbG9hZF9kYXRhc2V0CmZyb20gbWxydW4uYXJ0aWZhY3RzLm1hbmFnZXIgaW1wb3J0IEFydGlmYWN0LCBQbG90bHlBcnRpZmFjdApmcm9tIG1scnVuLmRhdGFzdG9yZSBpbXBvcnQgaXNfc3RvcmVfdXJpCmZyb20gbWxydW4uZnJhbWV3b3Jrcy5fY29tbW9uIGltcG9ydCBDb21tb25UeXBlcywgTUxSdW5JbnRlcmZhY2UKZnJvbSBtbHJ1bi51dGlscyBpbXBvcnQgbG9nZ2VyCmZyb20gcGVmdCBpbXBvcnQgKExvcmFDb25maWcsIFBlZnRNb2RlbCwgZ2V0X3BlZnRfbW9kZWwsCiAgICAgICAgICAgICAgICAgIHByZXBhcmVfbW9kZWxfZm9yX2tiaXRfdHJhaW5pbmcpCmZyb20gcGxvdGx5IGltcG9ydCBncmFwaF9vYmplY3RzIGFzIGdvCmZyb20gdHJhbnNmb3JtZXJzIGltcG9ydCAoQXV0b01vZGVsRm9yQ2F1c2FsTE0sIEF1dG9Ub2tlbml6ZXIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgQml0c0FuZEJ5dGVzQ29uZmlnLCBEYXRhQ29sbGF0b3JGb3JMYW5ndWFnZU1vZGVsaW5nLAogICAgICAgICAgICAgICAgICAgICAgICAgIFByZVRyYWluZWRNb2RlbCwgUHJlVHJhaW5lZFRva2VuaXplciwgVHJhaW5lciwKICAgICAgICAgICAgICAgICAgICAgICAgICBUcmFpbmVyQ2FsbGJhY2ssIFRyYWluZXJDb250cm9sLCBUcmFpbmVyU3RhdGUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgVHJhaW5pbmdBcmd1bWVudHMpCgpzdXBwb3J0ZWRfdGFza3MgPSBbCiAgICAicXVlc3Rpb24tYW5zd2VyaW5nIiwKICAgICJzdW1tYXJpemF0aW9uIiwKICAgICJ0YWJsZS1xdWVzdGlvbi1hbnN3ZXJpbmciLAogICAgInRleHQydGV4dC1nZW5lcmF0aW9uIiwKICAgICJ0ZXh0LWNsYXNzaWZpY2F0aW9uIiwKICAgICJzZW50aW1lbnQtYW5hbHlzaXMiLAogICAgInRleHQtZ2VuZXJhdGlvbiIsCiAgICAidG9rZW4tY2xhc3NpZmljYXRpb24iLAogICAgInRyYW5zbGF0aW9uIiwKICAgICJ0cmFuc2xhdGlvbl94eF90b195eSIsCl0KCgpjbGFzcyBDb25maWdLZXlzOgogICAgZGVlcHNwZWVkID0gImRlZXBzcGVlZCIKICAgIHF1YW50aXphdGlvbiA9ICJxdWFudGl6YXRpb24iCiAgICBsb3JhID0gImxvcmEiCiAgICB0cmFpbmluZyA9ICJ0cmFpbmluZyIKICAgIHRva2VuaXplcl9wcmV0cmFpbmVkID0gInRva2VuaXplcl9wcmV0cmFpbmVkIgogICAgbW9kZWxfcHJldHJhaW5lZCA9ICJtb2RlbF9wcmV0cmFpbmVkIgogICAgZGF0YV9jb2xsYXRvciA9ICJkYXRhX2NvbGxhdG9yIgoKCiMgLS0tLS0tLS0tLS0tLS0tLS0tLS0tLWZyb20gTUxSVU4tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQpjbGFzcyBIRlRyYWluZXJNTFJ1bkludGVyZmFjZShNTFJ1bkludGVyZmFjZSwgQUJDKToKICAgICIiIgogICAgVGhpcyBpcyB0ZW1wb3JhcnkgYW5kIHdpbGwgYmUgYnVpbHQgaW4gbWxydW4gMS41LjAKICAgIEludGVyZmFjZSBmb3IgYWRkaW5nIE1MUnVuIGZlYXR1cmVzIGZvciB0ZW5zb3JmbG93IGtlcmFzIEFQSS4KICAgICIiIgoKICAgICMgTUxSdW5zIGNvbnRleHQgZGVmYXVsdCBuYW1lOgogICAgREVGQVVMVF9DT05URVhUX05BTUUgPSAibWxydW4taHVnZ2luZ2ZhY2UiCgogICAgIyBBdHRyaWJ1dGVzIHRvIHJlcGxhY2Ugc28gdGhlIE1MUnVuIGludGVyZmFjZSB3aWxsIGJlIGZ1bGx5IGVuYWJsZWQuCiAgICBfUkVQTEFDRURfTUVUSE9EUyA9IFsKICAgICAgICAidHJhaW4iLAogICAgICAgICMgImV2YWx1YXRlIgogICAgXQoKICAgIEBjbGFzc21ldGhvZAogICAgZGVmIGFkZF9pbnRlcmZhY2UoCiAgICAgICAgY2xzLAogICAgICAgIG9iajogVHJhaW5lciwKICAgICAgICByZXN0b3JhdGlvbjogQ29tbW9uVHlwZXMuTUxSdW5JbnRlcmZhY2VSZXN0b3JhdGlvblR5cGUgPSBOb25lLAogICAgKToKICAgICAgICBzdXBlcihIRlRyYWluZXJNTFJ1bkludGVyZmFjZSwgY2xzKS5hZGRfaW50ZXJmYWNlKAogICAgICAgICAgICBvYmo9b2JqLCByZXN0b3JhdGlvbj1yZXN0b3JhdGlvbgogICAgICAgICkKCiAgICBAY2xhc3NtZXRob2QKICAgIGRlZiBtbHJ1bl90cmFpbihjbHMpOgogICAgICAgIGRlZiB3cmFwcGVyKHNlbGY6IFRyYWluZXIsICphcmdzLCAqKmt3YXJncyk6CiAgICAgICAgICAgICMgUmVzdG9yZSB0aGUgZXZhbHVhdGlvbiBtZXRob2QgYXMgYHRyYWluYCB3aWxsIHVzZSBpdDoKICAgICAgICAgICAgIyBjbHMuX3Jlc3RvcmVfYXR0cmlidXRlKG9iaj1zZWxmLCBhdHRyaWJ1dGVfbmFtZT0iZXZhbHVhdGUiKQoKICAgICAgICAgICAgIyBDYWxsIHRoZSBvcmlnaW5hbCBmaXQgbWV0aG9kOgogICAgICAgICAgICByZXN1bHQgPSBzZWxmLm9yaWdpbmFsX3RyYWluKCphcmdzLCAqKmt3YXJncykKCiAgICAgICAgICAgICMgUmVwbGFjZSB0aGUgZXZhbHVhdGlvbiBtZXRob2QgYWdhaW46CiAgICAgICAgICAgICMgY2xzLl9yZXBsYWNlX2Z1bmN0aW9uKG9iaj1zZWxmLCBmdW5jdGlvbl9uYW1lPSJldmFsdWF0ZSIpCgogICAgICAgICAgICByZXR1cm4gcmVzdWx0CgogICAgICAgIHJldHVybiB3cmFwcGVyCgoKY2xhc3MgTUxSdW5DYWxsYmFjayhUcmFpbmVyQ2FsbGJhY2spOgogICAgIiIiCiAgICBUaGlzIGlzIHRlbXBvcmFyeSBhbmQgd2lsbCBiZSBidWlsdCBpbiBtbHJ1biAxLjUuMAogICAgQ2FsbGJhY2sgZm9yIGNvbGxlY3RpbmcgbG9ncyBkdXJpbmcgdHJhaW5pbmcgLyBldmFsdWF0aW9uIG9mIHRoZSBgVHJhaW5lcmAgQVBJLgogICAgIiIiCgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsCiAgICAgICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHggPSBOb25lLAogICAgICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICAgICAgdGFnOiBzdHIgPSAiIiwKICAgICAgICBsYWJlbHM6IERpY3Rbc3RyLCBzdHJdID0gTm9uZSwKICAgICAgICBleHRyYV9kYXRhOiBkaWN0ID0gTm9uZSwKICAgICk6CiAgICAgICAgc3VwZXIoKS5fX2luaXRfXygpCgogICAgICAgICMgU3RvcmUgdGhlIGNvbmZpZ3VyYXRpb25zOgogICAgICAgIHNlbGYuX2NvbnRleHQgPSAoCiAgICAgICAgICAgIGNvbnRleHQKICAgICAgICAgICAgaWYgY29udGV4dCBpcyBub3QgTm9uZQogICAgICAgICAgICBlbHNlIG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KCIuL21scnVuLWh1Z2dpbmdmYWNlIikKICAgICAgICApCiAgICAgICAgc2VsZi5fbW9kZWxfbmFtZSA9IG1vZGVsX25hbWUKICAgICAgICBzZWxmLl90YWcgPSB0YWcKICAgICAgICBzZWxmLl9sYWJlbHMgPSBsYWJlbHMKICAgICAgICBzZWxmLl9leHRyYV9kYXRhID0gZXh0cmFfZGF0YSBpZiBleHRyYV9kYXRhIGlzIG5vdCBOb25lIGVsc2Uge30KCiAgICAgICAgIyBTZXQgdXAgdGhlIGxvZ2dpbmcgbW9kZToKICAgICAgICBzZWxmLl9pc190cmFpbmluZyA9IEZhbHNlCiAgICAgICAgc2VsZi5fc3RlcHM6IExpc3RbTGlzdFtpbnRdXSA9IFtdCiAgICAgICAgc2VsZi5fbWV0cmljX3Njb3JlczogRGljdFtzdHIsIExpc3RbZmxvYXRdXSA9IHt9CiAgICAgICAgc2VsZi5fYXJ0aWZhY3RzOiBEaWN0W3N0ciwgQXJ0aWZhY3RdID0ge30KCiAgICBkZWYgb25fZXBvY2hfYmVnaW4oCiAgICAgICAgc2VsZiwKICAgICAgICBhcmdzOiBUcmFpbmluZ0FyZ3VtZW50cywKICAgICAgICBzdGF0ZTogVHJhaW5lclN0YXRlLAogICAgICAgIGNvbnRyb2w6IFRyYWluZXJDb250cm9sLAogICAgICAgICoqa3dhcmdzLAogICAgKToKICAgICAgICBpZiBub3Qgc3RhdGUuaXNfd29ybGRfcHJvY2Vzc196ZXJvOgogICAgICAgICAgICByZXR1cm4KICAgICAgICBzZWxmLl9zdGVwcy5hcHBlbmQoW10pCgogICAgZGVmIG9uX2Vwb2NoX2VuZCgKICAgICAgICBzZWxmLAogICAgICAgIGFyZ3M6IFRyYWluaW5nQXJndW1lbnRzLAogICAgICAgIHN0YXRlOiBUcmFpbmVyU3RhdGUsCiAgICAgICAgY29udHJvbDogVHJhaW5lckNvbnRyb2wsCiAgICAgICAgKiprd2FyZ3MsCiAgICApOgogICAgICAgIGlmIG5vdCBzdGF0ZS5pc193b3JsZF9wcm9jZXNzX3plcm86CiAgICAgICAgICAgIHJldHVybgogICAgICAgIHNlbGYubG9nX21ldHJpY3MoKQoKICAgIGRlZiBvbl9sb2coCiAgICAgICAgc2VsZiwKICAgICAgICBhcmdzOiBUcmFpbmluZ0FyZ3VtZW50cywKICAgICAgICBzdGF0ZTogVHJhaW5lclN0YXRlLAogICAgICAgIGNvbnRyb2w6IFRyYWluZXJDb250cm9sLAogICAgICAgIGxvZ3M6IERpY3Rbc3RyLCBmbG9hdF0gPSBOb25lLAogICAgICAgICoqa3dhcmdzLAogICAgKToKICAgICAgICBpZiBub3Qgc3RhdGUuaXNfd29ybGRfcHJvY2Vzc196ZXJvOgogICAgICAgICAgICByZXR1cm4KICAgICAgICByZWNlbnRfbG9ncyA9IHN0YXRlLmxvZ19oaXN0b3J5Wy0xXS5jb3B5KCkKCiAgICAgICAgcmVjZW50X2xvZ3MucG9wKCJlcG9jaCIpCiAgICAgICAgY3VycmVudF9zdGVwID0gaW50KHJlY2VudF9sb2dzLnBvcCgic3RlcCIpKQogICAgICAgIGlmIGN1cnJlbnRfc3RlcCBub3QgaW4gc2VsZi5fc3RlcHNbLTFdOgogICAgICAgICAgICBzZWxmLl9zdGVwc1stMV0uYXBwZW5kKGN1cnJlbnRfc3RlcCkKCiAgICAgICAgZm9yIG1ldHJpY19uYW1lLCBtZXRyaWNfc2NvcmUgaW4gcmVjZW50X2xvZ3MuaXRlbXMoKToKICAgICAgICAgICAgaWYgbWV0cmljX25hbWUuc3RhcnRzd2l0aCgidHJhaW5fIik6CiAgICAgICAgICAgICAgICBpZiBtZXRyaWNfbmFtZS5zcGxpdCgidHJhaW5fIilbMV0gbm90IGluIHNlbGYuX21ldHJpY19zY29yZXM6CiAgICAgICAgICAgICAgICAgICAgc2VsZi5fbWV0cmljX3Njb3Jlc1ttZXRyaWNfbmFtZV0gPSBbbWV0cmljX3Njb3JlXQogICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgaWYgbWV0cmljX25hbWUgbm90IGluIHNlbGYuX21ldHJpY19zY29yZXM6CiAgICAgICAgICAgICAgICBzZWxmLl9tZXRyaWNfc2NvcmVzW21ldHJpY19uYW1lXSA9IFtdCiAgICAgICAgICAgIHNlbGYuX21ldHJpY19zY29yZXNbbWV0cmljX25hbWVdLmFwcGVuZChtZXRyaWNfc2NvcmUpCgogICAgZGVmIG9uX3RyYWluX2JlZ2luKAogICAgICAgIHNlbGYsCiAgICAgICAgYXJnczogVHJhaW5pbmdBcmd1bWVudHMsCiAgICAgICAgc3RhdGU6IFRyYWluZXJTdGF0ZSwKICAgICAgICBjb250cm9sOiBUcmFpbmVyQ29udHJvbCwKICAgICAgICAqKmt3YXJncywKICAgICk6CiAgICAgICAgaWYgbm90IHN0YXRlLmlzX3dvcmxkX3Byb2Nlc3NfemVybzoKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgc2VsZi5faXNfdHJhaW5pbmcgPSBUcnVlCgogICAgZGVmIG9uX3RyYWluX2VuZCgKICAgICAgICBzZWxmLAogICAgICAgIGFyZ3M6IFRyYWluaW5nQXJndW1lbnRzLAogICAgICAgIHN0YXRlOiBUcmFpbmVyU3RhdGUsCiAgICAgICAgY29udHJvbDogVHJhaW5lckNvbnRyb2wsCiAgICAgICAgbW9kZWw6IFByZVRyYWluZWRNb2RlbCA9IE5vbmUsCiAgICAgICAgdG9rZW5pemVyOiBQcmVUcmFpbmVkVG9rZW5pemVyID0gTm9uZSwKICAgICAgICAqKmt3YXJncywKICAgICk6CiAgICAgICAgaWYgbm90IHN0YXRlLmlzX3dvcmxkX3Byb2Nlc3NfemVybzoKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgc2VsZi5sb2dfbWV0cmljcygpCgogICAgZGVmIG9uX2V2YWx1YXRlKAogICAgICAgIHNlbGYsCiAgICAgICAgYXJnczogVHJhaW5pbmdBcmd1bWVudHMsCiAgICAgICAgc3RhdGU6IFRyYWluZXJTdGF0ZSwKICAgICAgICBjb250cm9sOiBUcmFpbmVyQ29udHJvbCwKICAgICAgICAqKmt3YXJncywKICAgICk6CiAgICAgICAgaWYgbm90IHN0YXRlLmlzX3dvcmxkX3Byb2Nlc3NfemVybzoKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgc2VsZi5sb2dfbWV0cmljcygpCgogICAgICAgIGlmIHNlbGYuX2lzX3RyYWluaW5nOgogICAgICAgICAgICByZXR1cm4KCiAgICBkZWYgbG9nX21ldHJpY3Moc2VsZik6CiAgICAgICAgZm9yIG1ldHJpY19uYW1lLCBtZXRyaWNfc2NvcmVzIGluIHNlbGYuX21ldHJpY19zY29yZXMuaXRlbXMoKToKICAgICAgICAgICAgc2VsZi5fY29udGV4dC5sb2dfcmVzdWx0KGtleT1tZXRyaWNfbmFtZSwgdmFsdWU9bWV0cmljX3Njb3Jlc1stMV0pCiAgICAgICAgICAgIGlmIGxlbihtZXRyaWNfc2NvcmVzKSA+IDE6CiAgICAgICAgICAgICAgICBzZWxmLmxvZ19tZXRyaWNfcGxvdChuYW1lPW1ldHJpY19uYW1lLCBzY29yZXM9bWV0cmljX3Njb3JlcykKICAgICAgICBzZWxmLl9jb250ZXh0LmNvbW1pdChjb21wbGV0ZWQ9RmFsc2UpCgogICAgZGVmIGxvZ19tZXRyaWNfcGxvdChzZWxmLCBuYW1lOiBzdHIsIHNjb3JlczogTGlzdFtmbG9hdF0pOgogICAgICAgICMgSW5pdGlhbGl6ZSBhIHBsb3RseSBmaWd1cmU6CiAgICAgICAgbWV0cmljX2ZpZ3VyZSA9IGdvLkZpZ3VyZSgpCgogICAgICAgICMgQWRkIHRpdGxlczoKICAgICAgICBtZXRyaWNfZmlndXJlLnVwZGF0ZV9sYXlvdXQoCiAgICAgICAgICAgIHRpdGxlPW5hbWUuY2FwaXRhbGl6ZSgpLnJlcGxhY2UoIl8iLCAiICIpLAogICAgICAgICAgICB4YXhpc190aXRsZT0iU2FtcGxlcyIsCiAgICAgICAgICAgIHlheGlzX3RpdGxlPSJTY29yZXMiLAogICAgICAgICkKCiAgICAgICAgIyBEcmF3OgogICAgICAgIG1ldHJpY19maWd1cmUuYWRkX3RyYWNlKAogICAgICAgICAgICBnby5TY2F0dGVyKHg9bnAuYXJhbmdlKGxlbihzY29yZXMpKSwgeT1zY29yZXMsIG1vZGU9ImxpbmVzIikKICAgICAgICApCgogICAgICAgICMgQ3JlYXRlIHRoZSBwbG90bHkgYXJ0aWZhY3Q6CiAgICAgICAgYXJ0aWZhY3RfbmFtZSA9IGYie25hbWV9X3Bsb3QiCiAgICAgICAgYXJ0aWZhY3QgPSBQbG90bHlBcnRpZmFjdChrZXk9YXJ0aWZhY3RfbmFtZSwgZmlndXJlPW1ldHJpY19maWd1cmUpCiAgICAgICAgc2VsZi5fYXJ0aWZhY3RzW2FydGlmYWN0X25hbWVdID0gc2VsZi5fY29udGV4dC5sb2dfYXJ0aWZhY3QoYXJ0aWZhY3QpCgoKZGVmIGFwcGx5X21scnVuKAogICAgdHJhaW5lcjogdHJhbnNmb3JtZXJzLlRyYWluZXIsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgdGFnOiBzdHIgPSAiIiwKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4ID0gTm9uZSwKICAgIGF1dG9fbG9nOiBib29sID0gVHJ1ZSwKICAgIGxhYmVsczogRGljdFtzdHIsIHN0cl0gPSBOb25lLAogICAgZXh0cmFfZGF0YTogZGljdCA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgVGhpcyBpcyB0ZW1wb3JhcnkgYW5kIHdpbGwgYmUgYnVpbHQgaW4gbWxydW4gMS41LjAKICAgICIiIgogICAgIyBHZXQgcGFyYW1ldGVycyBkZWZhdWx0czoKICAgIGlmIGNvbnRleHQgaXMgTm9uZToKICAgICAgICBjb250ZXh0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgoSEZUcmFpbmVyTUxSdW5JbnRlcmZhY2UuREVGQVVMVF9DT05URVhUX05BTUUpCgogICAgSEZUcmFpbmVyTUxSdW5JbnRlcmZhY2UuYWRkX2ludGVyZmFjZShvYmo9dHJhaW5lcikKCiAgICBpZiBhdXRvX2xvZzoKICAgICAgICB0cmFpbmVyLmFkZF9jYWxsYmFjaygKICAgICAgICAgICAgTUxSdW5DYWxsYmFjaygKICAgICAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICAgICAgICAgIHRhZz10YWcsCiAgICAgICAgICAgICAgICBsYWJlbHM9bGFiZWxzLAogICAgICAgICAgICAgICAgZXh0cmFfZGF0YT1leHRyYV9kYXRhLAogICAgICAgICAgICApCiAgICAgICAgKQoKCiMgLS0tLS0tLS0tLS0tLS0tLS0tLS0tLWVuZCBmcm9tIE1MUlVOLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0KCgpkZWYgX3ByaW50X3RyYWluYWJsZV9wYXJhbWV0ZXJzKG1vZGVsKToKICAgICIiIgogICAgUHJpbnRzIHRoZSBudW1iZXIgb2YgdHJhaW5hYmxlIHBhcmFtZXRlcnMgaW4gdGhlIG1vZGVsLgogICAgIiIiCiAgICB0cmFpbmFibGVfcGFyYW1zID0gMAogICAgYWxsX3BhcmFtID0gMAogICAgZm9yIF8sIHBhcmFtIGluIG1vZGVsLm5hbWVkX3BhcmFtZXRlcnMoKToKICAgICAgICBhbGxfcGFyYW0gKz0gcGFyYW0ubnVtZWwoKQogICAgICAgIGlmIHBhcmFtLnJlcXVpcmVzX2dyYWQ6CiAgICAgICAgICAgIHRyYWluYWJsZV9wYXJhbXMgKz0gcGFyYW0ubnVtZWwoKQogICAgcHJpbnQoCiAgICAgICAgZiJ0cmFpbmFibGUgcGFyYW1zOiB7dHJhaW5hYmxlX3BhcmFtc30gfHwgYWxsIHBhcmFtczoge2FsbF9wYXJhbX0gfHwgdHJhaW5hYmxlJToiCiAgICAgICAgZiIgezEwMCAqIHRyYWluYWJsZV9wYXJhbXMgLyBhbGxfcGFyYW19IgogICAgKQoKCiMgZGVmYXVsdCBjb25maWdzCiMgd2lsbCBiZSB1c2VkIGlmIHVzZXIgcHJvdmlkZXMgIlRydWUiIHdpdGggY29uZmlnIG5hbWUgYXMgaW5wdXQKUVVBTlRJWkFUSU9OX0NPTkZJRyA9IHRyYW5zZm9ybWVycy5CaXRzQW5kQnl0ZXNDb25maWcoCiAgICBsb2FkX2luXzRiaXQ9VHJ1ZSwKICAgIGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQ9VHJ1ZSwKICAgIGJuYl80Yml0X3F1YW50X3R5cGU9Im5mNCIsCiAgICBibmJfNGJpdF9jb21wdXRlX2R0eXBlPXRvcmNoLmJmbG9hdDE2LAopCgpMT1JBX0NPTkZJRyA9IHBlZnQuTG9yYUNvbmZpZygKICAgIHI9OCwKICAgIGxvcmFfYWxwaGE9MzIsCiAgICB0YXJnZXRfbW9kdWxlcz1bInF1ZXJ5X2tleV92YWx1ZSJdLAogICAgbG9yYV9kcm9wb3V0PTAuMDUsCiAgICBiaWFzPSJub25lIiwKICAgIHRhc2tfdHlwZT0iQ0FVU0FMX0xNIiwKKQoKREVFUFNQRUVEX0NPTkZJRyA9IHsKICAgICJ0cmFpbl9taWNyb19iYXRjaF9zaXplX3Blcl9ncHUiOiAiYXV0byIsCiAgICAiZnAxNiI6IHsiZW5hYmxlZCI6IFRydWV9LAogICAgImF1dG90dW5pbmciOiB7CiAgICAgICAgImVuYWJsZWQiOiBUcnVlLAogICAgICAgICJhcmdfbWFwcGluZ3MiOiB7CiAgICAgICAgICAgICJ0cmFpbl9taWNyb19iYXRjaF9zaXplX3Blcl9ncHUiOiAiLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUiLAogICAgICAgICAgICAiZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzICI6ICItLWdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyIsCiAgICAgICAgfSwKICAgIH0sCiAgICAiemVyb19vcHRpbWl6YXRpb24iOiB7CiAgICAgICAgInN0YWdlIjogMiwKICAgIH0sCn0KCgpkZWYgX3VwZGF0ZV9jb25maWcoc3JjOiBkaWN0LCBkc3Q6IGRpY3QpOgogICAgIiIiCiAgICB1cGRhdGUgY29uZmlncyBhY2NvcmRpbmcgdG8gdXNlciwgdGhpcyB3YXkgdGhlIHVzZXIgY2FuIGFkZC9tb2RpZnkgdmFsdWVzIGluIGRlZmF1bHQgY29uZmlncyBmb3IgZS5nLgoKICAgIGdvZXMgb3ZlciBhbGwgY29uZmlncyBhbmQgY29ycmVzcG9uZGluZyBwcmVmaXhlcywgY29sbGVjdCBhbGwgdGhlIGtleXMgZnJvbSB0aGUgZ2l2ZW4gZGljdCB0aGF0IHN0YXJ0CiAgICAgd2l0aCB0aGUgcHJlZml4IGFuZCBhZGQgdGhlbSB0byBhcHByb3ByaWF0ZSBjb25maWcKCiAgICA6cGFyYW0gc3JjOiBkaWN0IG9mIGFsbCBjYW5kaWRhdGUgdmFsdWVzIHRvIHVwZGF0ZSBkaWN0LgogICAgOnBhcmFtIGRzdDogZGljdCBjb250YWluaW5nIGFsbCBjb25maWdzIHRvIHVwZGF0ZS4KICAgICIiIgoKICAgIGZvciBjb25maWdfbmFtZSwgY29uZmlnIGluIGRzdC5pdGVtcygpOgoKICAgICAgICAjIElmIGdpdmVuIFRydWUgd2UgdXNlIGRlZmF1bHQgZGljdAogICAgICAgICMgQ2FuIGFsc28gYmUgRmFsc2Ugb3IgYSBjb25maWcgZGljdCBnaXZlbiBmcm9tIHVzZXIsIHNvIHdlIGNoZWNrIHNwZWNpZmljYWxseSBmbyBUcnVlCiAgICAgICAgaWYgY29uZmlnIGlzIFRydWUgYW5kIGNvbmZpZ19uYW1lID09ICJxdWFudGl6YXRpb24iOgogICAgICAgICAgICBjb25maWcgPSBRVUFOVElaQVRJT05fQ09ORklHCgogICAgICAgIGlmIGNvbmZpZyBpcyBUcnVlIGFuZCBjb25maWdfbmFtZSA9PSAibG9yYSI6CiAgICAgICAgICAgIGNvbmZpZyA9IExPUkFfQ09ORklHCgogICAgICAgIGlmIGNvbmZpZyBpcyBUcnVlIGFuZCBjb25maWdfbmFtZSA9PSAiZGVlcHNwZWVkIjoKICAgICAgICAgICAgY29uZmlnID0gREVFUFNQRUVEX0NPTkZJRwoKICAgICAgICAjIGluIHNvbWUgY2FzZXMgd2UgY2FuIGdldCBhIGJvb2xlYW4gdmFsdWUsIGluIHRoYXQgY2FzZSBubyBuZWVkIHRvIGxvb2sgZm9yIGFyZ3MKICAgICAgICBpZiBpc2luc3RhbmNlKGNvbmZpZywgYm9vbCk6CiAgICAgICAgICAgIGNvbmZpZyA9IE5vbmUKCiAgICAgICAgZWxpZiBpc2luc3RhbmNlKGNvbmZpZywgZGljdCk6CiAgICAgICAgICAgIGZvciBrZXksIHZhbCBpbiBzcmMuaXRlbXMoKToKICAgICAgICAgICAgICAgIGlmIGtleS5zdGFydHN3aXRoKGNvbmZpZ19uYW1lKToKICAgICAgICAgICAgICAgICAgICBjb25maWdba2V5LnJlcGxhY2UoZiJ7Y29uZmlnX25hbWV9XyIsICIiKV0gPSB2YWwKCiAgICAgICAgIyB1cGRhdGUgYnkgY29uZmlnIG5hbWUKICAgICAgICBlbHNlOgogICAgICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCk6CiAgICAgICAgICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChjb25maWdfbmFtZSk6CiAgICAgICAgICAgICAgICAgICAgc2V0YXR0cihjb25maWcsIGtleS5yZXBsYWNlKGYie2NvbmZpZ19uYW1lfV8iLCAiIiksIHZhbCkKCiAgICAgICAgZHN0LnVwZGF0ZSh7Y29uZmlnX25hbWU6IGNvbmZpZ30pCgoKZGVmIF9nZXRfY2xhc3Nfb2JqZWN0KGNsYXNzX3BhdGg6IHN0cikgLT4gdHlwZToKICAgICIiIgogICAgZ2l2ZW4gYSBmdWxsIGNsYXNzIG5hbWUsIHRoaXMgZnVuY3Rpb24gcmV0dXJucyB0aGUgY29ycmVjdCBjbGFzcwoKICAgIDpwYXJhbSBjbGFzc19wYXRoOiBhIGZ1bGwgY2xhc3MgbmFtZSAoZXguICd0cmFuc2Zvcm1lcnMuQXV0b01vZGVsRm9yQ2F1c2FsTE0nKQoKICAgIDpyZXR1cm4gdGhlIHdhbnRlZCBjbGFzcyBvYmplY3QKICAgICIiIgogICAgbW9kdWxlX3BhdGgsIGNsYXNzX25hbWUgPSBjbGFzc19wYXRoLnJzcGxpdCgiLiIsIDEpCiAgICBtb2R1bGUgPSBpbXBvcnRsaWIuaW1wb3J0X21vZHVsZShtb2R1bGVfcGF0aCkKICAgIHJldHVybiBnZXRhdHRyKG1vZHVsZSwgY2xhc3NfbmFtZSkKCgpkZWYgX3NldF9tb2RlbF9hbmRfdG9rZW5pemVyKAogICAgbW9kZWw6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSwKICAgIHRva2VuaXplcjogVW5pb25bc3RyLCBMaXN0W3N0cl1dLAogICAgdGFzazogc3RyLAogICAgZnJhbWV3b3JrOiBzdHIsCiAgICBsb3JhX2NvbmZpZzogZGljdCwKICAgIHF1YW50aXphdGlvbl9jb25maWc6IGRpY3QsCiAgICB1c2VfY3VkYTogYm9vbCwKICAgIHRva2VuaXplcl9wcmV0cmFpbmVkX2NvbmZpZywKICAgIG1vZGVsX3ByZXRyYWluZWRfY29uZmlnLAogICAgZGV2aWNlX21hcDogc3RyLAopOgogICAgIiIiCiAgICBnZXQgdGhlIGNvcnJlY3QgbW9kZWwgYW5kIHRva2VuaXplciBhY2NvcmRpbmcgdG8gZ2l2ZW4gdXNlciBpbnB1dHMKCiAgICA6cGFyYW0gbW9kZWw6IGEgdHVwbGUgY29udGFpbmluZyBtb2RlbCBuYW1lIGFuZCBjbGFzcywgb3Igc3RyIHdpdGggbW9kZWwgbmFtZSBvciBwYXRoCiAgICA6cGFyYW0gdG9rZW5pemVyOiBhIHR1cGxlIGNvbnRhaW5pbmcgdG9rZW5pemVyIG5hbWUgYW5kIGNsYXNzLCBvciBzdHIgd2l0aCB0b2tlbml6ZXIgbmFtZSBvciBwYXRoCiAgICA6cGFyYW0gdGFzazogYSBzdXBwb3J0ZWQgbmxwIHRhc2ssIHVzZWQgdG8gY2hvb3NlIG1vZGVsIGlmIG5vdCBwcm92aWRlZAogICAgOnBhcmFtIGZyYW1ld29yazogcHQgb3IgdGYKICAgIDpwYXJhbSBsb3JhX2NvbmZpZzogbG9yYSBjb25maWcgb3IgTm9uZSwgdG8gbG9hZCBtb2RlbCBpbiBhcHByb3ByaWF0ZSB3YXkKICAgIDpwYXJhbSBxdWFudGl6YXRpb25fY29uZmlnOiBxdWFudGl6YXRpb24gY29uZmlnIG9yIE5vbmUsIHRvIGxvYWQgbW9kZWwgaW4gYXBwcm9wcmlhdGUgd2F5CiAgICA6cGFyYW0gdXNlX2N1ZGE6IHVzZSBncHUgb3Igbm90CiAgICA6cGFyYW0gdG9rZW5pemVyX3ByZXRyYWluZWRfY29uZmlnOiBjb25maWcgdG8gbG9hZCB0aGUgcHJldHJhaW5lZCB0b2tlbml6ZXIKICAgIDpwYXJhbSBtb2RlbF9wcmV0cmFpbmVkX2NvbmZpZzogY29uZmlnIHRvIGxvYWQgdGhlIHByZXRyYWluZWQgbW9kZWwKICAgIDpwYXJhbSBkZXZpY2VfbWFwOiBhIGRldmljZSBtYXAgZm9yIG1vZGVsIHRyYWluaW5nIGlmIHVzaW5nIG51bWJlciBvZiBncHUncwoKICAgIDpyZXR1cm5zOiBtb2RlbCBhbmQgdG9rZW5pemVyCiAgICAiIiIKICAgICMgaWYgdGFzayBpcyBub3Qgc3VwcG9ydGVkIGFuZCBubyBtb2RlbCB3YXMgZ2l2ZW4gd2UgY2FuJ3QgY2hvb3NlIG9uZQogICAgaWYgdGFzayBhbmQgdGFzayBub3QgaW4gc3VwcG9ydGVkX3Rhc2tzIGFuZCBub3QgbW9kZWw6CiAgICAgICAgbG9nZ2VyLmVycm9yKCJ1bnN1cHBvcnRlZCB0YXNrIG9wdGlvbiBjaG9zZW4iKQogICAgICAgIHJhaXNlCgogICAgIyBsb2FkIG1vZGVsIGZyb20gc3RvcmUKICAgIGlmIGlzaW5zdGFuY2UobW9kZWwsIHN0cikgYW5kIGlzX3N0b3JlX3VyaShtb2RlbCk6CiAgICAgICAgcGFzcwogICAgICAgICMgVE9ETzogbG9hZCBib3RoIG1vZGVsIGFuZCB0b2tlbml6ZXIgYW5kIHJldHVybiwgbmVlZCBndXkncyBoZWxwCgogICAgIyBpZiBpdCdzIGEgdHVwbGUgdGhlbSB3ZSBhc3N1bWUgaXQgY29udGFpbnMgb2YgYm90aCBuYW1lIGFuZCBjbGFzcwogICAgaWYgaXNpbnN0YW5jZShtb2RlbCwgbGlzdCk6CiAgICAgICAgbW9kZWxfbmFtZSwgbW9kZWxfY2xhc3MgPSBtb2RlbAogICAgICAgIG1vZGVsX2NsYXNzID0gX2dldF9jbGFzc19vYmplY3QobW9kZWxfY2xhc3MpCgogICAgIyBpbiB0aGUgY2FzZSB3ZSBkb24ndCBnZXQgdGhlIG1vZGVsIGNsYXNzIHdlIG5lZWQgdGhlIHRhc2sgaW4gb3JkZXIgdG8gY2hvb3NlIHRoZSBjb3JyZWN0IG1vZGVsCiAgICBlbHNlOgogICAgICAgIGlmIHRhc2sgaXMgTm9uZToKICAgICAgICAgICAgbG9nZ2VyLmVycm9yKCJ0YXNrIG11c3QgYmUgY2hvc2VuIGluIG9yZGVyIHRvIGRldGVybWluZSB0aGUgY29ycmVjdCBtb2RlbCIpCiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigKICAgICAgICAgICAgICAgICJ0aGlzIGZ1bmN0aW9uIHJlcXVpcmVzIGVpdGhlciBhIHN1cHBvcnRlZCB0YXNrIG9yIGEgbW9kZWwgYW5kIG1vZGVsIGNsYXNzIHRvIGJlIGNob3NlbiIKICAgICAgICAgICAgKQoKICAgICAgICBfLCBhdmFpbGFibGVfY2xhc3NlcywgdGFza19vcHRpb25zID0gdHJhbnNmb3JtZXJzLnBpcGVsaW5lcy5jaGVja190YXNrKHRhc2spCgogICAgICAgIGlmIGlzaW5zdGFuY2UobW9kZWwsIHN0cik6CiAgICAgICAgICAgIG1vZGVsX25hbWUgPSBtb2RlbAoKICAgICAgICAjIGlmIG1vZGVsIGlzIG5vdCBnaXZlbiwgd2UgdGFrZSB0aGUgZGVmYXVsdCBtb2RlbCBmb3IgdGhlIGdpdmVuIHRhc2sKICAgICAgICBlbHNlOgogICAgICAgICAgICBtb2RlbF9uYW1lLCBfID0gdHJhbnNmb3JtZXJzLnBpcGVsaW5lcy5nZXRfZGVmYXVsdF9tb2RlbF9hbmRfcmV2aXNpb24oCiAgICAgICAgICAgICAgICBhdmFpbGFibGVfY2xhc3NlcywgZnJhbWV3b3JrLCB0YXNrX29wdGlvbnMKICAgICAgICAgICAgKQogICAgICAgIGlmIG5vdCBhdmFpbGFibGVfY2xhc3Nlcy5nZXQoZnJhbWV3b3JrLCB0dXBsZSgpKToKICAgICAgICAgICAgbG9nZ2VyLmVycm9yKAogICAgICAgICAgICAgICAgImdpdmVuIHRhc2sncyBkZWZhdWx0IG1vZGVsIGlzIG5vdCBzdXBwb3J0ZWQgaW4gc3BlY2lmaWVkIGZyYW1ld29yayIKICAgICAgICAgICAgKQogICAgICAgICAgICByYWlzZSBFeGNlcHRpb24oCiAgICAgICAgICAgICAgICAidGhpcyBmdW5jdGlvbiByZXF1aXJlcyBlaXRoZXIgYSBzdXBwb3J0ZWQgdGFzayBvciBhIG1vZGVsIGFuZCBtb2RlbCBjbGFzcyB0byBiZSBjaG9zZW4iCiAgICAgICAgICAgICkKCiAgICAgICAgbW9kZWxfY2xhc3MgPSBhdmFpbGFibGVfY2xhc3Nlc1tmcmFtZXdvcmtdWzBdCgogICAgIyBsb2FkIHRoZSBwcmV0cmFpbmVkIG1vZGVsCiAgICBpZiB1c2VfY3VkYToKICAgICAgICBkZXZpY2VfbWFwID0gZGV2aWNlX21hcAogICAgZWxzZToKICAgICAgICBkZXZpY2VfbWFwID0gTm9uZQoKICAgIG1vZGVsID0gbW9kZWxfY2xhc3MuZnJvbV9wcmV0cmFpbmVkKAogICAgICAgIG1vZGVsX25hbWUsCiAgICAgICAgcXVhbnRpemF0aW9uX2NvbmZpZz1xdWFudGl6YXRpb25fY29uZmlnLAogICAgICAgIGRldmljZV9tYXA9ZGV2aWNlX21hcCwKICAgICAgICAqKm1vZGVsX3ByZXRyYWluZWRfY29uZmlnLAogICAgKQoKICAgICMgSWYgcXVhbnRpemF0aW9uIGNvbmZpZyBpcyBnaXZlbiB3ZSB3aWxsIGxvYWQgYSBxdWFudGl6ZWQgbW9kZWwsIGlmIG5vdCBhIHJlZ3VsYXIgb25lCiAgICBpZiBxdWFudGl6YXRpb25fY29uZmlnOgogICAgICAgIG1vZGVsLmdyYWRpZW50X2NoZWNrcG9pbnRpbmdfZW5hYmxlKCkKICAgICAgICBtb2RlbCA9IHBlZnQucHJlcGFyZV9tb2RlbF9mb3Jfa2JpdF90cmFpbmluZyhtb2RlbCkKCiAgICAjIElmIGxvcmEgY29uZmlnIHdhcyBnaXZlbiB3ZSB3YW50IHRvIGRvIGxvcmEgZmluZSB0dW5lLCB3ZSB1cGRhdGUgbW9kZWwgaGVyZQogICAgaWYgbG9yYV9jb25maWc6CiAgICAgICAgbW9kZWwgPSBwZWZ0LmdldF9wZWZ0X21vZGVsKG1vZGVsLCBsb3JhX2NvbmZpZykKCiAgICAjIGlmIG5vdCBzcGVjaWZpZWQgd2UgY2hvb3NlIHRoZSBkZWZhdWx0IHRva2VuaXplciB0aGF0IGNvcnJlc3BvbmRpbmcgdG8gdGhlIG1vZGVsCiAgICBpZiB0b2tlbml6ZXIgaXMgTm9uZToKICAgICAgICB0b2tlbml6ZXIgPSB0cmFuc2Zvcm1lcnMuQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSkKICAgICAgICByZXR1cm4gbW9kZWxfbmFtZSwgbW9kZWwsIHRva2VuaXplcgoKICAgIGlmIGlzaW5zdGFuY2UodG9rZW5pemVyLCBzdHIpOgogICAgICAgIHRva2VuaXplcl9uYW1lID0gdG9rZW5pemVyCiAgICAgICAgdG9rZW5pemVyX2NsYXNzID0gdHJhbnNmb3JtZXJzLkF1dG9Ub2tlbml6ZXIKCiAgICAjIGlmIGl0J3Mgbm90IGEgc3RyIHRoZW4gaXQncyBhIHR1cGxlIG9mIGJvdGggbmFtZSBhbmQgY2xhc3MKICAgIGVsc2U6CiAgICAgICAgdG9rZW5pemVyX25hbWUsIHRva2VuaXplcl9jbGFzcyA9IHRva2VuaXplcgogICAgICAgIHRva2VuaXplcl9jbGFzcyA9IF9nZXRfY2xhc3Nfb2JqZWN0KHRva2VuaXplcl9jbGFzcykKCiAgICB0b2tlbml6ZXIgPSB0b2tlbml6ZXJfY2xhc3MuZnJvbV9wcmV0cmFpbmVkKAogICAgICAgIHRva2VuaXplcl9uYW1lLCAqKnRva2VuaXplcl9wcmV0cmFpbmVkX2NvbmZpZwogICAgKQoKICAgIHRva2VuaXplci5wYWRfdG9rZW4gPSB0b2tlbml6ZXIuZW9zX3Rva2VuCgogICAgcmV0dXJuIG1vZGVsX25hbWUsIG1vZGVsLCB0b2tlbml6ZXIKCgpkZWYgX2RhdGFzZXRfbG9hZGVyKGRhdGFzZXQ6IHN0ciwgaXNfdHJhaW46IGJvb2wgPSBUcnVlLCAqKmt3YXJncykgLT4gRGF0YXNldDoKICAgICIiIgogICAgbG9hZHMgdGhlIHNwZWNpZmljIGRhdGFzZXQgcHJvdmlkZWQgYnkgdGhlIHVzZXIKCiAgICA6cGFyYW0gZGF0YXNldDogbmFtZSBvciBwYXRoIG9mIGRhdGFzZXQgdG8gbG9hZAogICAgOnBhcmFtIGlzX3RyYWluOiBib29sIHRoYXQgaW5kaWNhdGVzIHRoZSBwdXJwb3NlIG9mIHRoZSBkYXRhc2V0CiAgICA6cGFyYW0ga3dhcmdzOiBvdGhlciBrd2FyZ3MgZm9yIGxvYWRpbmcgdGhlIGRhdGFzZXQKCiAgICA6cmV0dXJuczogbG9hZGVkIGRhdGFzZXQKICAgICIiIgogICAgIyBpZiBzcGxpdCBpbiBrd2FyZ3MgdGhlbiB0aGUgdXNlciBkZWNpZGVzIGhvdyB0byBzcGxpdCB0aGUgZGF0YXNldAogICAgaWYgInNwbGl0IiBpbiBrd2FyZ3M6CiAgICAgICAgcmV0dXJuIGxvYWRfZGF0YXNldChkYXRhc2V0LCAqKmt3YXJncykKCiAgICAjIGlmIGl0J3MgYSBkYXRhc2V0IGZvciB0cmFpbiB3ZSBzcGxpdCB3aXRoIHRyYWluCiAgICBpZiBpc190cmFpbjoKICAgICAgICByZXR1cm4gbG9hZF9kYXRhc2V0KGRhdGFzZXQsIHNwbGl0PSJ0cmFpbiIsICoqa3dhcmdzKQoKICAgICMgaWYgaXQncyBldmFsIGRhdGFzZXQsIHRoZW4gYSBsb3Qgb2YgbmFtZXMgYXJlIGFjY2VwdGFibGUgZm9yIHRoZSBzZXQgYW5kIHdlIGNoZWNrIGFsbCBvZiB0aGVtCiAgICBkYXRhc2V0ID0gbG9hZF9kYXRhc2V0KGRhdGFzZXQsICoqa3dhcmdzKQogICAgaWYgInRlc3QiIGluIGRhdGFzZXQ6CiAgICAgICAgcmV0dXJuIGRhdGFzZXQuZ2V0KCJ0ZXN0IikKICAgIGVsaWYgImV2YWwiIGluIGRhdGFzZXQ6CiAgICAgICAgcmV0dXJuIGRhdGFzZXQuZ2V0KCJldmFsIikKICAgIGVsaWYgInZhbGlkYXRpb24iIGluIGRhdGFzZXQ6CiAgICAgICAgcmV0dXJuIGRhdGFzZXQuZ2V0KCJ2YWxpZGF0aW9uIikKCgpkZWYgX3ByZXBhcmVfZGF0YXNldCgKICAgIHRyYWluX2RhdGFzZXQ6IHN0ciwKICAgIGV2YWxfZGF0YXNldDogc3RyLAogICAgdHJhaW5fbG9hZF9kYXRhc2V0X2t3YXJncywKICAgIGV2YWxfbG9hZF9kYXRhc2V0X2t3YXJncywKICAgIHRva2VuaXplciwKICAgIGRhdGFzZXRfY29sdW1uc190b190cmFpbjogVW5pb25bc3RyLCBsaXN0XSwKKSAtPiAoRGF0YXNldCwgVW5pb25bRGF0YXNldCwgTm9uZV0pOgogICAgIiIiCiAgICBMb2FkcyB0aGUgdHJhaW4gYW5kIGV2YWwgZGF0YXNldHMgKGlmIHByb3ZpZGVkKSBwYXNzZXMgdGhlbSB0aHJvdWdoIHRoZSB0b2tlbml6ZXIgYW5kCiAgICByZXR1cm5zIHRoZW0gcmVhZHkgdG8gdXNlIGluIHRyYWluaW5nCgogICAgOnBhcmFtIHRyYWluX2RhdGFzZXQ6IHRoZSBuYW1lIG9yIHBhdGggdG8gdGhlIHRyYWluIGRhdGFzZXQKICAgIDpwYXJhbSBldmFsX2RhdGFzZXQ6IHRoZSBuYW1lIG9yIHBhdGggdG8gdGhlIGV2YWwgZGF0YXNldAogICAgOnBhcmFtIGRhdGFzZXRfY29sdW1uc190b190cmFpbjogd2hpY2ggY29sdW1ucyB0byBwYXNzIHRvIHRoZSBtb2RlbCBhcyBpbnB1dHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChuZWVkIHRvIHBhc3MgdGhyb3VnaCB0aGUgdG9rZW5pemVyIGZpcnN0KQogICAgOnBhcmFtIHRyYWluX2xvYWRfZGF0YXNldF9rd2FyZ3M6IGt3YXJncyBmb3IgZGF0YXNldCBsb2FkaW5nCiAgICA6cGFyYW0gZXZhbF9sb2FkX2RhdGFzZXRfa3dhcmdzOiBrd2FyZ3MgZm9yIGRhdGFzZXQgbG9hZGluZwogICAgOnBhcmFtIHRva2VuaXplcjogdGhlIHRva2VuaXplciB0byBwYXNzIHRoZSBkYXRhIHRocm91Z2gKCiAgICA6cmV0dXJuczogdG9rZW5pemVkIGRhdGFzZXRzCiAgICAiIiIKICAgIGlmIG5vdCB0b2tlbml6ZXIucGFkX3Rva2VuOgogICAgICAgIHRva2VuaXplci5wYWRfdG9rZW4gPSB0b2tlbml6ZXIuZW9zX3Rva2VuCgogICAgIyB3ZSB0YWtlIGNvbCBuYW1lL3MgaW4gYSBsaXN0IGZvciBlYXN5IGdlbmVyYWxpemF0aW9uCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXRfY29sdW1uc190b190cmFpbiwgc3RyKToKICAgICAgICBkYXRhc2V0X2NvbHVtbnNfdG9fdHJhaW4gPSBbZGF0YXNldF9jb2x1bW5zX3RvX3RyYWluXQoKICAgIGlmIGlzaW5zdGFuY2UodHJhaW5fZGF0YXNldCwgbWxydW4uZGF0YXN0b3JlLkRhdGFJdGVtKToKICAgICAgICB0cmFpbl9kYXRhc2V0ID0gRGF0YXNldC5mcm9tX3BhbmRhcyh0cmFpbl9kYXRhc2V0LmFzX2RmKCkpCiAgICAgICAgcmV0dXJuICgKICAgICAgICAgICAgdHJhaW5fZGF0YXNldC5tYXAoCiAgICAgICAgICAgICAgICBsYW1iZGEgZXhhbXBsZXM6IHRva2VuaXplcigKICAgICAgICAgICAgICAgICAgICAqW2V4YW1wbGVzW2NvbF0gZm9yIGNvbCBpbiBkYXRhc2V0X2NvbHVtbnNfdG9fdHJhaW5dLAogICAgICAgICAgICAgICAgICAgIHRydW5jYXRpb249VHJ1ZSwKICAgICAgICAgICAgICAgICAgICBwYWRkaW5nPVRydWUsCiAgICAgICAgICAgICAgICApLAogICAgICAgICAgICAgICAgYmF0Y2hlZD1UcnVlLAogICAgICAgICAgICApLAogICAgICAgICAgICBOb25lLAogICAgICAgICkKCiAgICAjIExvYWQgZGF0YXNldHMKICAgICMgaWYgcHJvdmlkZWQgdHdvIHBhdGhzL25hbWVzIHdlIGxvYWQgZWFjaCBzZXBhcmF0ZWx5IHVzaW5nIGRlc2lnbmF0ZWQgZnVuYwogICAgaWYgZXZhbF9kYXRhc2V0OgogICAgICAgIHRyYWluX2RhdGFzZXQgPSBfZGF0YXNldF9sb2FkZXIoCiAgICAgICAgICAgIGRhdGFzZXQ9dHJhaW5fZGF0YXNldCwgaXNfdHJhaW49VHJ1ZSwgKip0cmFpbl9sb2FkX2RhdGFzZXRfa3dhcmdzCiAgICAgICAgKQogICAgICAgIGV2YWxfZGF0YXNldCA9IF9kYXRhc2V0X2xvYWRlcigKICAgICAgICAgICAgZGF0YXNldD1ldmFsX2RhdGFzZXQsIGlzX3RyYWluPUZhbHNlLCAqKmV2YWxfbG9hZF9kYXRhc2V0X2t3YXJncwogICAgICAgICkKCiAgICAjIGlmIG9ubHkgb24gcGF0aCBpcyBnaXZlbiB0aGVuIHdlIG11c3QgY2hlY2sgaWYgaXQgY29udGFpbnMgYm90aCBkYXRhc2V0IG9yIGlmIG9ubHkgb25lIHNob3VsZCBiZSB1c2VkCiAgICBlbHNlOgogICAgICAgIGRhdGFzZXQgPSBsb2FkX2RhdGFzZXQodHJhaW5fZGF0YXNldCwgKip0cmFpbl9sb2FkX2RhdGFzZXRfa3dhcmdzKQogICAgICAgIGlmICJ0cmFpbiIgaW4gZGF0YXNldDoKICAgICAgICAgICAgdHJhaW5fZGF0YXNldCA9IGRhdGFzZXQuZ2V0KCJ0cmFpbiIpCiAgICAgICAgICAgIGlmICJ0ZXN0IiBpbiBkYXRhc2V0OgogICAgICAgICAgICAgICAgZXZhbF9kYXRhc2V0ID0gZGF0YXNldC5nZXQoInRlc3QiKQogICAgICAgICAgICBlbGlmICJldmFsIiBpbiBkYXRhc2V0OgogICAgICAgICAgICAgICAgZXZhbF9kYXRhc2V0ID0gZGF0YXNldC5nZXQoImV2YWwiKQogICAgICAgICAgICBlbGlmICJ2YWxpZGF0aW9uIiBpbiBkYXRhc2V0OgogICAgICAgICAgICAgICAgZXZhbF9kYXRhc2V0ID0gZGF0YXNldC5nZXQoInZhbGlkYXRpb24iKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgIyBvbmx5IHRyYWluIGRhdGFzZXQgZ2l2ZW4sIHRva2VuaXplIGFuZCByZXR1cm4gaXQKICAgICAgICAgICAgICAgIHJldHVybiAoCiAgICAgICAgICAgICAgICAgICAgdHJhaW5fZGF0YXNldC5tYXAoCiAgICAgICAgICAgICAgICAgICAgICAgIGxhbWJkYSBleGFtcGxlczogdG9rZW5pemVyKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgKltleGFtcGxlc1tjb2xdIGZvciBjb2wgaW4gZGF0YXNldF9jb2x1bW5zX3RvX3RyYWluXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRydW5jYXRpb249VHJ1ZSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhZGRpbmc9VHJ1ZSwKICAgICAgICAgICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgICAgICAgICAgYmF0Y2hlZD1UcnVlLAogICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICAgICAgTm9uZSwKICAgICAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICBsb2dnZXIuZXJyb3IoInRyYWluIGRhdGFzZXQgaXMgbWFuZGF0b3J5IikKICAgICAgICAgICAgcmFpc2UgS2V5RXJyb3IoIm5vIHRyYWluIGRhdGFzZXQgZm91bmQgaW4gZ2l2ZW4gZGF0YXNldCIpCgogICAgIyBUb2tlbml6ZSB0aGUgZGF0YSBzbyB0aGUgbW9kZWwgY2FuIHVuZGVyc3RhbmQgaXQKICAgIHRva2VuaXplZF90cmFpbl9kYXRhc2V0ID0gdHJhaW5fZGF0YXNldC5tYXAoCiAgICAgICAgbGFtYmRhIGV4YW1wbGVzOiB0b2tlbml6ZXIoCiAgICAgICAgICAgICpbZXhhbXBsZXNbY29sXSBmb3IgY29sIGluIGRhdGFzZXRfY29sdW1uc190b190cmFpbl0sCiAgICAgICAgICAgIHRydW5jYXRpb249VHJ1ZSwKICAgICAgICAgICAgcGFkZGluZz1UcnVlLAogICAgICAgICksCiAgICAgICAgYmF0Y2hlZD1UcnVlLAogICAgKQoKICAgIHRva2VuaXplZF9ldmFsX2RhdGFzZXQgPSBldmFsX2RhdGFzZXQubWFwKAogICAgICAgIGxhbWJkYSBleGFtcGxlczogdG9rZW5pemVyKAogICAgICAgICAgICAqW2V4YW1wbGVzW2NvbF0gZm9yIGNvbCBpbiBkYXRhc2V0X2NvbHVtbnNfdG9fdHJhaW5dLAogICAgICAgICAgICB0cnVuY2F0aW9uPVRydWUsCiAgICAgICAgICAgIHBhZGRpbmc9VHJ1ZSwKICAgICAgICApLAogICAgICAgIGJhdGNoZWQ9VHJ1ZSwKICAgICkKCiAgICByZXR1cm4gdG9rZW5pemVkX3RyYWluX2RhdGFzZXQsIHRva2VuaXplZF9ldmFsX2RhdGFzZXQKCgpkZWYgZmluZXR1bmVfbGxtKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICB0cmFpbl9kYXRhc2V0OiBVbmlvbltzdHIsIG1scnVuLmRhdGFzdG9yZS5EYXRhSXRlbV0sCiAgICBldmFsX2RhdGFzZXQ6IHN0ciA9IE5vbmUsCiAgICB0cmFpbl9sb2FkX2RhdGFzZXRfa3dhcmdzOiBkaWN0ID0ge30sCiAgICBldmFsX2xvYWRfZGF0YXNldF9rd2FyZ3M6IGRpY3QgPSB7fSwKICAgIGRhdGFzZXRfY29sdW1uc190b190cmFpbjogVW5pb25bc3RyLCBsaXN0XSA9ICJ0ZXh0IiwKICAgIG1vZGVsOiBVbmlvbltzdHIsIExpc3Rbc3RyXV0gPSAiaHVnZ2luZ2ZhY2UtbW9kZWwiLAogICAgdG9rZW5pemVyOiBVbmlvbltzdHIsIExpc3Rbc3RyXV0gPSBOb25lLAogICAgZGVlcHNwZWVkX2NvbmZpZzogVW5pb25bZGljdCwgYm9vbF0gPSBGYWxzZSwKICAgIHF1YW50aXphdGlvbl9jb25maWc6IFVuaW9uW2RpY3QsIGJvb2xdID0gRmFsc2UsCiAgICBsb3JhX2NvbmZpZzogVW5pb25bZGljdCwgYm9vbF0gPSBGYWxzZSwKICAgIHRyYWluaW5nX2NvbmZpZzogZGljdCA9IHt9LAogICAgbW9kZWxfcHJldHJhaW5lZF9jb25maWc6IGRpY3QgPSB7fSwKICAgIHRva2VuaXplcl9wcmV0cmFpbmVkX2NvbmZpZzogZGljdCA9IHt9LAogICAgZGF0YV9jb2xsYXRvcl9jb25maWc6IGRpY3QgPSB7fSwKICAgIHRhc2s6IHN0ciA9ICJ0ZXh0LWdlbmVyYXRpb24iLAogICAgdXNlX2N1ZGE6IGJvb2wgPSBUcnVlLAogICAgZnJhbWV3b3JrOiBzdHIgPSAicHQiLAogICAgZGV2aWNlX21hcDogc3RyID0gImF1dG8iLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEZpbmUtdHVuZXMgYSBMYW5ndWFnZSBNb2RlbCAoTExNKSBvbiBhIHNwZWNpZmljIHRhc2sgdXNpbmcgdGhlIHByb3ZpZGVkIGRhdGFzZXQuCiAgICAgVGhlIGZ1bmN0aW9uIHRha2VzIHZhcmlvdXMgY29uZmlndXJhdGlvbiBwYXJhbWV0ZXJzIHRvIGN1c3RvbWl6ZSB0aGUgdHJhaW5pbmcgcHJvY2VzcwogICAgIGFuZCBhZGFwdCB0aGUgbW9kZWwgdG8gc3BlY2lmaWMgdGFza3MgdXNpbmcgYSBwcm92aWRlZCBkYXRhc2V0LgoKICAgIDpwYXJhbSBjb250ZXh0OiBtbHJ1biBjb250ZXh0IGluIG9yZGVyIHRvIGxvZyB0cmFpbmVkIG1vZGVsCiAgICA6cGFyYW0gZGF0YXNldF9jb2x1bW5zX3RvX3RyYWluOiB3aGljaCBjb2x1bW5zIHRvIHBhc3MgdG8gdGhlIG1vZGVsIGFzIGlucHV0cwogICAgOnBhcmFtIGV2YWxfbG9hZF9kYXRhc2V0X2t3YXJnczoga3dhcmdzIGZvciBkYXRhc2V0IGxvYWRpbmcKICAgIDpwYXJhbSB0cmFpbl9sb2FkX2RhdGFzZXRfa3dhcmdzOiBrd2FyZ3MgZm9yIGRhdGFzZXQgbG9hZGluZwogICAgOnBhcmFtIGZyYW1ld29yazogcHQgb3QgdGYKICAgIDpwYXJhbSB1c2VfY3VkYTogdXNlIGdwdSBvciBub3QKICAgIDpwYXJhbSB0b2tlbml6ZXJfcHJldHJhaW5lZF9jb25maWc6IGNvbmZpZyB0byBsb2FkIHRoZSBwcmV0cmFpbmVkIHRva2VuaXplcgogICAgOnBhcmFtIG1vZGVsX3ByZXRyYWluZWRfY29uZmlnOiBjb25maWcgdG8gbG9hZCB0aGUgcHJldHJhaW5lZCBtb2RlbAogICAgOnBhcmFtIHRva2VuaXplcjogYSB0dXBsZSBjb250YWluaW5nIHRva2VuaXplciBuYW1lIGFuZCBjbGFzcywgb3Igc3RyIHdpdGggdG9rZW5pemVyIG5hbWUgb3IgcGF0aAogICAgOnBhcmFtIG1vZGVsOiBhIHR1cGxlIGNvbnRhaW5pbmcgbW9kZWwgbmFtZSBhbmQgY2xhc3MsIG9yIHN0ciB3aXRoIG1vZGVsIG5hbWUgb3IgcGF0aAogICAgOnBhcmFtIHRyYWluX2RhdGFzZXQ6IFRoZSB0cmFpbiBkYXRhc2V0IHVzZWQgZm9yIGZpbmUtdHVuaW5nIHRoZSBsYW5ndWFnZSBtb2RlbC4KICAgIDpwYXJhbSBldmFsX2RhdGFzZXQ6IFRoZSBldmFsIGRhdGFzZXQgdXNlZCBmb3IgZXZhbHVhdGUgdGhlIGxhbmd1YWdlIG1vZGVsIGR1cmluZyB0cmFpbmluZy4KICAgIDpwYXJhbSBkZWVwc3BlZWRfY29uZmlnOiBDb25maWd1cmF0aW9uIG9wdGlvbnMgZm9yIERlZXBTcGVlZCAob3B0aW9uYWwpLgogICAgOnBhcmFtIHF1YW50aXphdGlvbl9jb25maWc6IENvbmZpZ3VyYXRpb24gb3B0aW9ucyBmb3IgbW9kZWwgcXVhbnRpemF0aW9uIChvcHRpb25hbCkuCiAgICA6cGFyYW0gbG9yYV9jb25maWc6IENvbmZpZ3VyYXRpb24gb3B0aW9ucyBmb3IgTG93LVJhbmsgQXBwcm94aW1hdGlvbiAoTG9SQSkgKG9wdGlvbmFsKS4KICAgIDpwYXJhbSB0cmFpbmluZ19jb25maWc6IENvbmZpZ3VyYXRpb24gb3B0aW9ucyBzcGVjaWZpYyB0byB0aGUgZmluZS10dW5pbmcgdHJhaW5pbmcgcHJvY2VzcyAob3B0aW9uYWwpLgogICAgOnBhcmFtIGRhdGFfY29sbGF0b3JfY29uZmlnOiBDb25maWd1cmF0aW9uIG9wdGlvbnMgZm9yIGRhdGEgY29sbGF0aW9uIGR1cmluZyB0cmFpbmluZyAob3B0aW9uYWwpLgogICAgOnBhcmFtIHRhc2s6IEEgZGVzY3JpcHRpb24gb2YgdGhlIHNwZWNpZmljIHRhc2sgdGhlIG1vZGVsIGlzIGJlaW5nIGZpbmUtdHVuZWQgZm9yLgogICAgOnBhcmFtIGt3YXJnczogQWRkaXRpb25hbCBrZXl3b3JkIGFyZ3VtZW50cy4KICAgICIiIgoKICAgICMgVE9ETzogbWF0Y2ggZm9yd2FyZC5rZXl3b3JkIHRvIGRhdGFzZXQua2V5d29yZCAtIGNoZWNrIGlmIHJlbGV2YW50IGluIG5ldyBkZXNpZ24KICAgICMgVE9ETzogYWRkIHdhcm5pbmcgZm9yIGxhYmVsLCBhbmQgYWRkIG9wdGlvbiB0byBtb2RpZnkgZGF0YXNldCBjb2wgbmFtZXMgLSBjaGVjayBpZiByZWxldmFudCBpbiBuZXcgZGVzaWduCgogICAgIyBMb29rIGZvciB1cGRhdGVzIHRvIGNvbmZpZ3MgZ2l2ZW4gaW4ga3dhcmdzCiAgICBjb25maWdzID0gewogICAgICAgIENvbmZpZ0tleXMuZGVlcHNwZWVkOiBkZWVwc3BlZWRfY29uZmlnLAogICAgICAgIENvbmZpZ0tleXMucXVhbnRpemF0aW9uOiBxdWFudGl6YXRpb25fY29uZmlnLAogICAgICAgIENvbmZpZ0tleXMubG9yYTogbG9yYV9jb25maWcsCiAgICAgICAgQ29uZmlnS2V5cy50cmFpbmluZzogdHJhaW5pbmdfY29uZmlnLAogICAgICAgIENvbmZpZ0tleXMubW9kZWxfcHJldHJhaW5lZDogbW9kZWxfcHJldHJhaW5lZF9jb25maWcsCiAgICAgICAgQ29uZmlnS2V5cy50b2tlbml6ZXJfcHJldHJhaW5lZDogdG9rZW5pemVyX3ByZXRyYWluZWRfY29uZmlnLAogICAgICAgIENvbmZpZ0tleXMuZGF0YV9jb2xsYXRvcjogZGF0YV9jb2xsYXRvcl9jb25maWcsCiAgICB9CiAgICBfdXBkYXRlX2NvbmZpZyhkc3Q9Y29uZmlncywgc3JjPWt3YXJncykKCiAgICAjIGNoZWNrIGdwdSBwZXJtaXNzaW9uIGFuZCBhdmFpbGFiaWxpdHkKICAgIGlmIHVzZV9jdWRhOgogICAgICAgIGlmIHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCk6CiAgICAgICAgICAgICMgQ2xlYW4gZ3B1IGNhY2hlCiAgICAgICAgICAgIHRvcmNoLmN1ZGEuZW1wdHlfY2FjaGUoKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxvZ2dlci53YXJuaW5nKCIndXNlX2N1ZGEnIGlzIHNldCB0byBUcnVlLCBidXQgbm8gY3VkYSBkZXZpY2UgaXMgYXZhaWxhYmxlIikKCiAgICAjIGdldCBtb2RlbCBhbmQgdG9rZW5pemVyCiAgICBtb2RlbF9uYW1lLCBtb2RlbCwgdG9rZW5pemVyID0gX3NldF9tb2RlbF9hbmRfdG9rZW5pemVyKAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIHRva2VuaXplcj10b2tlbml6ZXIsCiAgICAgICAgdGFzaz10YXNrLAogICAgICAgIGZyYW1ld29yaz1mcmFtZXdvcmssCiAgICAgICAgbG9yYV9jb25maWc9Y29uZmlnc1tDb25maWdLZXlzLmxvcmFdLAogICAgICAgIHF1YW50aXphdGlvbl9jb25maWc9Y29uZmlnc1tDb25maWdLZXlzLnF1YW50aXphdGlvbl0sCiAgICAgICAgdXNlX2N1ZGE9dXNlX2N1ZGEsCiAgICAgICAgdG9rZW5pemVyX3ByZXRyYWluZWRfY29uZmlnPXRva2VuaXplcl9wcmV0cmFpbmVkX2NvbmZpZywKICAgICAgICBtb2RlbF9wcmV0cmFpbmVkX2NvbmZpZz1jb25maWdzW0NvbmZpZ0tleXMubW9kZWxfcHJldHJhaW5lZF0sCiAgICAgICAgZGV2aWNlX21hcD1kZXZpY2VfbWFwLAogICAgKQoKICAgICMgTG9hZCBkYXRhc2V0cwogICAgdG9rZW5pemVkX3RyYWluLCB0b2tlbml6ZWRfZXZhbCA9IF9wcmVwYXJlX2RhdGFzZXQoCiAgICAgICAgdHJhaW5fZGF0YXNldD10cmFpbl9kYXRhc2V0LAogICAgICAgIGV2YWxfZGF0YXNldD1ldmFsX2RhdGFzZXQsCiAgICAgICAgdHJhaW5fbG9hZF9kYXRhc2V0X2t3YXJncz10cmFpbl9sb2FkX2RhdGFzZXRfa3dhcmdzLAogICAgICAgIGV2YWxfbG9hZF9kYXRhc2V0X2t3YXJncz1ldmFsX2xvYWRfZGF0YXNldF9rd2FyZ3MsCiAgICAgICAgdG9rZW5pemVyPXRva2VuaXplciwKICAgICAgICBkYXRhc2V0X2NvbHVtbnNfdG9fdHJhaW49ZGF0YXNldF9jb2x1bW5zX3RvX3RyYWluLAogICAgKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgZGF0YSBjb2xsYXRvciBmb3IgdGhlIHRyYWluZXIgdG8gdXNlIGluIG9yZGVyIHRvIGNyZWF0ZSBiYXRjaGVzIG9mIGRhdGEKICAgIGRhdGFfY29sbGF0b3IgPSB0cmFuc2Zvcm1lcnMuRGF0YUNvbGxhdG9yRm9yTGFuZ3VhZ2VNb2RlbGluZygKICAgICAgICB0b2tlbml6ZXI9dG9rZW5pemVyLCBtbG09RmFsc2UsICoqZGF0YV9jb2xsYXRvcl9jb25maWcKICAgICkKCiAgICAjIEluaXRpYWxpemUgdHJhaW5pbmcga3dhcmdzIGZyb20gdXNlciBrd2FyZ3M6CiAgICB0cmFpbl9rd2FyZ3MgPSBjb25maWdzW0NvbmZpZ0tleXMudHJhaW5pbmddCgogICAgIyBJZiBkZWVwc3BlZWQgY29uZmlnIGdpdmVuIHdlIGFkZCBpdCB0byB0cmFpbmluZyBrd2FyZ3MKICAgIGlmIGNvbmZpZ3NbQ29uZmlnS2V5cy5kZWVwc3BlZWRdOgogICAgICAgIHRyYWluX2t3YXJnc1siZGVlcHNwZWVkIl0gPSBjb25maWdzW0NvbmZpZ0tleXMuZGVlcHNwZWVkXQoKICAgICMgVGFrZSBhIGxvb2sgYXQgdGhlIHRyYWluYWJsZSBwYXJhbWV0ZXJzIGluIHRoZSBtb2RlbAogICAgX3ByaW50X3RyYWluYWJsZV9wYXJhbWV0ZXJzKG1vZGVsKQoKICAgICMgUHJlcGFyaW5nIHRyYWluaW5nIGFyZ3VtZW50czoKICAgIHRyYWluaW5nX2FyZ3MgPSB0cmFuc2Zvcm1lcnMuVHJhaW5pbmdBcmd1bWVudHMoCiAgICAgICAgb3V0cHV0X2Rpcj10ZW1wZmlsZS5ta2R0ZW1wKCksCiAgICAgICAgKip0cmFpbl9rd2FyZ3MsCiAgICApCgogICAgdHJhaW5lciA9IHRyYW5zZm9ybWVycy5UcmFpbmVyKAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIHRyYWluX2RhdGFzZXQ9dG9rZW5pemVkX3RyYWluLAogICAgICAgIGV2YWxfZGF0YXNldD10b2tlbml6ZWRfZXZhbCwKICAgICAgICB0b2tlbml6ZXI9dG9rZW5pemVyLAogICAgICAgIGRhdGFfY29sbGF0b3I9ZGF0YV9jb2xsYXRvciwKICAgICAgICBhcmdzPXRyYWluaW5nX2FyZ3MsCiAgICApCgogICAgYXBwbHlfbWxydW4odHJhaW5lciwgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLnNwbGl0KCIvIilbLTFdKQogICAgbW9kZWwuY29uZmlnLnVzZV9jYWNoZSA9ICgKICAgICAgICBGYWxzZSAgIyBzaWxlbmNlIHRoZSB3YXJuaW5ncy4gUGxlYXNlIHJlLWVuYWJsZSBmb3IgaW5mZXJlbmNlIQogICAgKQoKICAgICMgQXBwbHkgdHJhaW5pbmcgd2l0aCBldmFsdWF0aW9uOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRyYWluaW5nICd7bW9kZWxfbmFtZX0nIikKICAgIHRyYWluZXIudHJhaW4oKQoKICAgIHRlbXBfZGlyZWN0b3J5ID0gdGVtcGZpbGUuVGVtcG9yYXJ5RGlyZWN0b3J5KCkubmFtZQogICAgdHJhaW5lci5zYXZlX21vZGVsKHRlbXBfZGlyZWN0b3J5KQoKICAgICMgWmlwIHRoZSBtb2RlbCBkaXJlY3Rvcnk6CiAgICBzaHV0aWwubWFrZV9hcmNoaXZlKAogICAgICAgIGJhc2VfbmFtZT0ibW9kZWwiLAogICAgICAgIGZvcm1hdD0iemlwIiwKICAgICAgICByb290X2Rpcj10ZW1wX2RpcmVjdG9yeSwKICAgICkKCiAgICAjIExvZyB0aGUgbW9kZWw6CiAgICBjb250ZXh0LmxvZ19tb2RlbCgKICAgICAgICBrZXk9Im1vZGVsIiwKICAgICAgICBkYl9rZXk9bW9kZWxfbmFtZS5zcGxpdCgiLyIpWy0xXSwKICAgICAgICBtb2RlbF9maWxlPSJtb2RlbC56aXAiLAogICAgICAgIHRhZz0iIiwKICAgICAgICBmcmFtZXdvcms9Ikh1Z2dpbmcgRmFjZSIsCiAgICApCgoKZGVmIGV2YWx1YXRlKAogICAgY29udGV4dCwKICAgIG1vZGVsX3BhdGgsCiAgICBkYXRhOiBwZC5EYXRhRnJhbWUsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgdG9rZW5pemVyX25hbWU6IHN0ciA9IE5vbmUsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgdGhlIG1vZGVsIHVzaW5nIHBlcnBsZXhpdHksIGZvciBtb3JlIGluZm9ybWF0aW9uIHZpc2l0OgogICAgaHR0cHM6Ly9odWdnaW5nZmFjZS5jby9kb2NzL3RyYW5zZm9ybWVycy9wZXJwbGV4aXR5CgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICBtbHJ1biBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGF0aDogIHBhdGggdG8gdGhlIG1vZGVsIGRpcmVjdG9yeQogICAgOnBhcmFtIGRhdGE6ICAgICAgICB0aGUgZGF0YSB0byBldmFsdWF0ZSB0aGUgbW9kZWwKICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgbmFtZSBvZiBiYXNlIG1vZGVsCiAgICA6cGFyYW0gdG9rZW5pemVyX25hbWU6IG5hbWUgb2YgYmFzZSB0b2tlbml6ZXIKICAgICIiIgogICAgIyBHZXQgdGhlIG1vZGVsIGFydGlmYWN0IGFuZCBmaWxlOgogICAgKAogICAgICAgIG1vZGVsX2ZpbGUsCiAgICAgICAgbW9kZWxfYXJ0aWZhY3QsCiAgICAgICAgZXh0cmFfZGF0YSwKICAgICkgPSBtbHJ1bi5hcnRpZmFjdHMuZ2V0X21vZGVsKG1vZGVsX3BhdGgpCgogICAgIyBSZWFkIHRoZSBuYW1lOgogICAgX21vZGVsX25hbWUgPSBtb2RlbF9hcnRpZmFjdC5zcGVjLmRiX2tleQoKICAgICMgRXh0cmFjdCBsb2dnZWQgbW9kZWwgZmlsZXM6CiAgICBtb2RlbF9kaXJlY3RvcnkgPSBvcy5wYXRoLmpvaW4ob3MucGF0aC5kaXJuYW1lKG1vZGVsX2ZpbGUpLCBfbW9kZWxfbmFtZSkKICAgIHdpdGggemlwZmlsZS5aaXBGaWxlKG1vZGVsX2ZpbGUsICJyIikgYXMgemlwX2ZpbGU6CiAgICAgICAgemlwX2ZpbGUuZXh0cmFjdGFsbChtb2RlbF9kaXJlY3RvcnkpCgogICAgIyBMb2FkaW5nIHRoZSBzYXZlZCBwcmV0cmFpbmVkIHRva2VuaXplciBhbmQgbW9kZWw6CiAgICBkYXRhc2V0ID0gRGF0YXNldC5mcm9tX3BhbmRhcyhkYXRhKQogICAgdG9rZW5pemVyID0gQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQodG9rZW5pemVyX25hbWUpCiAgICBwYWRfdG9rZW5faWQgPSB0b2tlbml6ZXIuZW9zX3Rva2VuX2lkCiAgICBtb2RlbCA9IEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCgKICAgICAgICBtb2RlbF9uYW1lLCBkZXZpY2VfbWFwPSJjdWRhOjAiLCB0cnVzdF9yZW1vdGVfY29kZT1UcnVlLCBsb2FkX2luXzhiaXQ9VHJ1ZQogICAgKQogICAgbW9kZWwgPSBQZWZ0TW9kZWwuZnJvbV9wcmV0cmFpbmVkKG1vZGVsLCBtb2RlbF9kaXJlY3RvcnkpCiAgICBtb2RlbC5ldmFsKCkKICAgIGVuY29kaW5ncyA9IHRva2VuaXplcigiXG5cbiIuam9pbihkYXRhc2V0WyJ0ZXh0Il1bOjVdKSwgcmV0dXJuX3RlbnNvcnM9InB0IikKCiAgICBtYXhfbGVuZ3RoID0gMTAyNAogICAgc3RyaWRlID0gNTEyCiAgICBzZXFfbGVuID0gZW5jb2RpbmdzLmlucHV0X2lkcy5zaXplKDEpCgogICAgbmxscyA9IFtdCiAgICBwcmV2X2VuZF9sb2MgPSAwCiAgICBmb3IgYmVnaW5fbG9jIGluIHJhbmdlKDAsIHNlcV9sZW4sIHN0cmlkZSk6CiAgICAgICAgZW5kX2xvYyA9IG1pbihiZWdpbl9sb2MgKyBtYXhfbGVuZ3RoLCBzZXFfbGVuKQogICAgICAgIHRyZ19sZW4gPSBlbmRfbG9jIC0gcHJldl9lbmRfbG9jICAjIG1heSBiZSBkaWZmZXJlbnQgZnJvbSBzdHJpZGUgb24gbGFzdCBsb29wCiAgICAgICAgaW5wdXRfaWRzID0gZW5jb2RpbmdzLmlucHV0X2lkc1s6LCBiZWdpbl9sb2M6ZW5kX2xvY10KICAgICAgICB0YXJnZXRfaWRzID0gaW5wdXRfaWRzLmNsb25lKCkKICAgICAgICB0YXJnZXRfaWRzWzosIDotdHJnX2xlbl0gPSAtMTAwCgogICAgICAgIHdpdGggdG9yY2gubm9fZ3JhZCgpOgogICAgICAgICAgICBvdXRwdXRzID0gbW9kZWwoaW5wdXRfaWRzLmN1ZGEoKSwgbGFiZWxzPXRhcmdldF9pZHMpCgogICAgICAgICAgICAjIGxvc3MgaXMgY2FsY3VsYXRlZCB1c2luZyBDcm9zc0VudHJvcHlMb3NzIHdoaWNoIGF2ZXJhZ2VzIG92ZXIgdmFsaWQgbGFiZWxzCiAgICAgICAgICAgICMgTi5CLiB0aGUgbW9kZWwgb25seSBjYWxjdWxhdGVzIGxvc3Mgb3ZlciB0cmdfbGVuIC0gMSBsYWJlbHMsIGJlY2F1c2UgaXQgaW50ZXJuYWxseSBzaGlmdHMgdGhlIGxhYmVscwogICAgICAgICAgICAjIHRvIHRoZSBsZWZ0IGJ5IDEuCiAgICAgICAgICAgIG5lZ19sb2dfbGlrZWxpaG9vZCA9IG91dHB1dHMubG9zcwoKICAgICAgICBubGxzLmFwcGVuZChuZWdfbG9nX2xpa2VsaWhvb2QpCgogICAgICAgIHByZXZfZW5kX2xvYyA9IGVuZF9sb2MKICAgICAgICBpZiBlbmRfbG9jID09IHNlcV9sZW46CiAgICAgICAgICAgIGJyZWFrCgogICAgcHBsID0gdG9yY2guZXhwKHRvcmNoLnN0YWNrKG5sbHMpLm1lYW4oKSkuaXRlbSgpCiAgICBjb250ZXh0LmxvZ19yZXN1bHQoInBlcnBsZXhpdHkiLCBwcGwpCg== commands: [] - code_origin: https://github.com/ZeevRispler/functions.git#a63a647cf6bc3015a8dcbd18903f9db44bdf0b66:/Users/Zeev_Rispler/PycharmProjects/functions/huggingface_auto_trainer/huggingface_auto_trainer.py - origin_filename: /Users/Zeev_Rispler/PycharmProjects/functions/huggingface_auto_trainer/huggingface_auto_trainer.py + code_origin: '' + origin_filename: '' requirements: [] entry_points: add_interface: @@ -25,183 +27,161 @@ spec: doc: '' parameters: - name: cls - default: '' - name: obj type: Trainer - default: '' - name: restoration type: MLRunInterfaceRestorationType default: null - outputs: - - default: '' + outputs: [] lineno: 70 + has_varargs: false + has_kwargs: false mlrun_train: name: mlrun_train doc: '' parameters: - name: cls - default: '' - outputs: - - default: '' + outputs: [] lineno: 80 + has_varargs: false + has_kwargs: false wrapper: name: wrapper doc: '' parameters: - name: self type: Trainer - default: '' - outputs: - - default: '' + outputs: [] lineno: 81 + has_varargs: true + has_kwargs: true on_epoch_begin: name: on_epoch_begin doc: '' parameters: - name: self - default: '' - name: args type: TrainingArguments - default: '' - name: state type: TrainerState - default: '' - name: control type: TrainerControl - default: '' - outputs: - - default: '' + outputs: [] lineno: 129 + has_varargs: false + has_kwargs: true on_epoch_end: name: on_epoch_end doc: '' parameters: - name: self - default: '' - name: args type: TrainingArguments - default: '' - name: state type: TrainerState - default: '' - name: control type: TrainerControl - default: '' - outputs: - - default: '' + outputs: [] lineno: 140 + has_varargs: false + has_kwargs: true on_log: name: on_log doc: '' parameters: - name: self - default: '' - name: args type: TrainingArguments - default: '' - name: state type: TrainerState - default: '' - name: control type: TrainerControl - default: '' - name: logs type: Dict[str, float] default: null - outputs: - - default: '' + outputs: [] lineno: 151 + has_varargs: false + has_kwargs: true on_train_begin: name: on_train_begin doc: '' parameters: - name: self - default: '' - name: args type: TrainingArguments - default: '' - name: state type: TrainerState - default: '' - name: control type: TrainerControl - default: '' - outputs: - - default: '' + outputs: [] lineno: 177 + has_varargs: false + has_kwargs: true on_train_end: name: on_train_end doc: '' parameters: - name: self - default: '' - name: args type: TrainingArguments - default: '' - name: state type: TrainerState - default: '' - name: control type: TrainerControl - default: '' - name: model type: PreTrainedModel default: null - name: tokenizer type: PreTrainedTokenizer default: null - outputs: - - default: '' + outputs: [] lineno: 188 + has_varargs: false + has_kwargs: true on_evaluate: name: on_evaluate doc: '' parameters: - name: self - default: '' - name: args type: TrainingArguments - default: '' - name: state type: TrainerState - default: '' - name: control type: TrainerControl - default: '' - outputs: - - default: '' + outputs: [] lineno: 201 + has_varargs: false + has_kwargs: true log_metrics: name: log_metrics doc: '' parameters: - name: self - default: '' - outputs: - - default: '' + outputs: [] lineno: 215 + has_varargs: false + has_kwargs: false log_metric_plot: name: log_metric_plot doc: '' parameters: - name: self - default: '' - name: name type: str - default: '' - name: scores type: List[float] - default: '' - outputs: - - default: '' + outputs: [] lineno: 222 + has_varargs: false + has_kwargs: false apply_mlrun: name: apply_mlrun doc: This is temporary and will be built in mlrun 1.5.0 parameters: - name: trainer type: Trainer - default: '' - name: model_name type: str default: null @@ -220,9 +200,10 @@ spec: - name: extra_data type: dict default: null - outputs: - - default: '' + outputs: [] lineno: 244 + has_varargs: false + has_kwargs: true finetune_llm: name: finetune_llm doc: "Fine-tunes a Language Model (LLM) on a specific task using the provided\ @@ -233,11 +214,9 @@ spec: - name: context type: MLClientCtx doc: mlrun context in order to log trained model - default: '' - name: train_dataset type: Union[str, mlrun.datastore.DataItem] doc: The train dataset used for fine-tuning the language model. - default: '' - name: eval_dataset type: str doc: The eval dataset used for evaluate the language model during training. @@ -306,9 +285,10 @@ spec: - name: device_map type: str default: auto - outputs: - - default: '' + outputs: [] lineno: 630 + has_varargs: false + has_kwargs: true evaluate: name: evaluate doc: 'Evaluating the model using perplexity, for more information visit: @@ -317,14 +297,11 @@ spec: parameters: - name: context doc: mlrun context - default: '' - name: model_path doc: path to the model directory - default: '' - name: data type: DataFrame doc: the data to evaluate the model - default: '' - name: model_name type: str doc: name of base model @@ -333,9 +310,10 @@ spec: type: str doc: name of base tokenizer default: null - outputs: - - default: '' + outputs: [] lineno: 784 + has_varargs: false + has_kwargs: false description: fine-tune llm model with ease default_handler: finetune_llm disable_auto_mount: false diff --git a/huggingface_auto_trainer/item.yaml b/huggingface_auto_trainer/item.yaml index e556c11df..b7c9bbcca 100644 --- a/huggingface_auto_trainer/item.yaml +++ b/huggingface_auto_trainer/item.yaml @@ -1,5 +1,7 @@ apiVersion: v1 categories: +- huggingface +- genai - machine-learning - model-training description: fine-tune llm model with ease @@ -22,4 +24,4 @@ spec: kind: job requirements: [] url: '' -version: 1.0.0 +version: 1.1.0 diff --git a/pii_recognizer/function.yaml b/pii_recognizer/function.yaml index 54b448d9c..069fa1ffe 100644 --- a/pii_recognizer/function.yaml +++ b/pii_recognizer/function.yaml @@ -2,13 +2,14 @@ kind: job metadata: name: pii-recognizer tag: '' - hash: b09b7b9a4ffd55088d665a0191055411e9198a2f + hash: 818930645d33704e9cada919769ee9d93cbb9434 project: '' labels: author: pgw categories: - machine-learning - data-preparation + - NLP spec: command: '' args: [] diff --git a/pii_recognizer/item.yaml b/pii_recognizer/item.yaml index 2f618febc..41ead33b6 100644 --- a/pii_recognizer/item.yaml +++ b/pii_recognizer/item.yaml @@ -2,6 +2,7 @@ apiVersion: v1 categories: - machine-learning - data-preparation + - NLP description: This function is used to recognize PII in a directory of text files doc: '' example: pii_recognizer.ipynb @@ -30,5 +31,5 @@ spec: - st-annotated-text - https://huggingface.co/beki/en_spacy_pii_distilbert/resolve/main/en_spacy_pii_distilbert-any-py3-none-any.whl url: '' -version: 0.2.0 +version: 0.3.0 test_valid: False diff --git a/pyannote_audio/function.yaml b/pyannote_audio/function.yaml index 2e84fbd92..30870afa2 100644 --- a/pyannote_audio/function.yaml +++ b/pyannote_audio/function.yaml @@ -2,14 +2,14 @@ kind: job metadata: name: pyannote-audio tag: '' - hash: c45be8d7f51f0b2203155b08c307814a2cb0ac78 + hash: aed670a0534ebf30690dd2af7acad35595c7d5b1 project: '' labels: author: guyl categories: - deep-learning - - Huggingface - - Audio + - huggingface + - audio spec: command: '' args: [] diff --git a/pyannote_audio/item.yaml b/pyannote_audio/item.yaml index 7133ceb41..b69add9e6 100644 --- a/pyannote_audio/item.yaml +++ b/pyannote_audio/item.yaml @@ -1,8 +1,8 @@ apiVersion: v1 categories: - deep-learning -- Huggingface -- Audio +- huggingface +- audio description: pyannote's speech diarization of audio files doc: '' example: pyannote_audio.ipynb @@ -27,4 +27,4 @@ spec: - torchaudio - tqdm url: '' -version: 1.1.0 +version: 1.2.0 diff --git a/question_answering/function.yaml b/question_answering/function.yaml index a33614153..7491b17e9 100644 --- a/question_answering/function.yaml +++ b/question_answering/function.yaml @@ -2,11 +2,13 @@ kind: job metadata: name: question-answering tag: '' - hash: 90e67d116b256a98da7d5819724e43df01d8b4eb + hash: aed62db95f17576c69b457767e3595c2de1d5465 project: '' labels: author: yonish categories: + - genai + - huggingface - machine-learning spec: command: '' diff --git a/question_answering/item.yaml b/question_answering/item.yaml index 58ab5cc36..56fc5a5ec 100755 --- a/question_answering/item.yaml +++ b/question_answering/item.yaml @@ -1,5 +1,7 @@ apiVersion: v1 categories: +- genai +- huggingface - machine-learning description: GenAI approach of question answering on a given data doc: '' @@ -24,4 +26,4 @@ spec: - torch - tqdm url: '' -version: 0.3.1 +version: 0.4.0 diff --git a/silero_vad/function.yaml b/silero_vad/function.yaml index 0b4ad422b..8ec121a6b 100644 --- a/silero_vad/function.yaml +++ b/silero_vad/function.yaml @@ -2,14 +2,14 @@ kind: job metadata: name: silero-vad tag: '' - hash: 61b7a70c167b7819481fdabf9350fc6fa344d2f5 + hash: 59336f808643a74f3a2c5d506977387010427208 project: '' labels: author: guyl categories: - deep-learning - - PyTorch - - Audio + - pytorch + - audio spec: command: '' args: [] diff --git a/silero_vad/item.yaml b/silero_vad/item.yaml index 17c8eb62c..9ce9a5d2e 100644 --- a/silero_vad/item.yaml +++ b/silero_vad/item.yaml @@ -1,8 +1,8 @@ apiVersion: v1 categories: - deep-learning -- PyTorch -- Audio +- pytorch +- audio description: Silero VAD (Voice Activity Detection) functions. doc: '' example: silero_vad.ipynb @@ -27,4 +27,4 @@ spec: - tqdm - onnxruntime url: '' -version: 1.2.0 +version: 1.3.0 diff --git a/structured_data_generator/function.yaml b/structured_data_generator/function.yaml index 6f2039e4b..1093e178b 100644 --- a/structured_data_generator/function.yaml +++ b/structured_data_generator/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: structured-data-generator tag: '' - hash: ac969f46aae91804024ea736856267c26578864b + hash: 44bb39f4bc55b38fc7ead1df24cb02bcf7f05bc9 project: '' labels: author: zeevr @@ -10,7 +10,7 @@ metadata: - machine-learning - data-preparation - data-generation - - GenAI + - genai spec: command: '' args: [] diff --git a/structured_data_generator/item.yaml b/structured_data_generator/item.yaml index 27e0e3fab..be2a2a948 100755 --- a/structured_data_generator/item.yaml +++ b/structured_data_generator/item.yaml @@ -3,7 +3,7 @@ categories: - machine-learning - data-preparation - data-generation -- GenAI +- genai description: GenAI approach of generating structured data according to a given schema doc: '' example: structured_data_generator.ipynb @@ -26,4 +26,4 @@ spec: - langchain - tqdm url: '' -version: 1.4.0 +version: 1.5.0 diff --git a/text_to_audio_generator/function.yaml b/text_to_audio_generator/function.yaml index df142d2ef..88ef9cb89 100644 --- a/text_to_audio_generator/function.yaml +++ b/text_to_audio_generator/function.yaml @@ -2,13 +2,14 @@ kind: job metadata: name: text-to-audio-generator tag: '' - hash: 534e34d316098dcb345860a786ea013102150e67 + hash: 89fcaf3fab53e7b7fbba448a5e65c253d7fa66ed project: '' labels: author: yonatans categories: - data-preparation - machine-learning + - pytorch spec: command: '' args: [] diff --git a/text_to_audio_generator/item.yaml b/text_to_audio_generator/item.yaml index 4784a80d2..efa8afc90 100644 --- a/text_to_audio_generator/item.yaml +++ b/text_to_audio_generator/item.yaml @@ -2,6 +2,7 @@ apiVersion: v1 categories: - data-preparation - machine-learning +- pytorch description: Generate audio file from text using different speakers doc: '' example: text_to_audio_generator.ipynb @@ -24,5 +25,5 @@ spec: - bark - torchaudio url: '' -version: 1.1.0 +version: 1.2.0 test_valid: True diff --git a/transcribe/function.yaml b/transcribe/function.yaml index 40dd2f0e6..d72751ad6 100644 --- a/transcribe/function.yaml +++ b/transcribe/function.yaml @@ -2,12 +2,14 @@ kind: job metadata: name: transcribe tag: '' - hash: 5cd620de67a936ee8a87cfc1f0b97e19730d0a69 + hash: 8810ac74045bd15cee15a2e4e89563e8e29908d3 project: '' labels: author: yonatans categories: - data-preparation + - genai + - huggingface - machine-learning spec: command: '' @@ -24,6 +26,7 @@ spec: - tqdm - torchaudio - torch + - accelerate entry_points: do_task: name: do_task diff --git a/transcribe/item.yaml b/transcribe/item.yaml index d53341ff2..7fddcf95e 100644 --- a/transcribe/item.yaml +++ b/transcribe/item.yaml @@ -1,6 +1,8 @@ apiVersion: v1 categories: - data-preparation +- genai +- huggingface - machine-learning description: Transcribe audio files into text files doc: '' @@ -27,4 +29,4 @@ spec: - torch - accelerate url: '' -version: 1.0.0 \ No newline at end of file +version: 1.1.0 \ No newline at end of file diff --git a/translate/function.yaml b/translate/function.yaml index 1a3fd7a88..bb1656103 100644 --- a/translate/function.yaml +++ b/translate/function.yaml @@ -2,13 +2,16 @@ kind: job metadata: name: translate tag: '' - hash: bc26313449cd13554a18106ed9893535fb79dd6e + hash: 7eedf684bcebfbfd964e5503afbb56335c8f4097 project: '' labels: author: guyl categories: - data-preparation + - huggingface - machine-learning + - deep-learning + - NLP spec: command: '' args: [] @@ -34,24 +37,27 @@ spec: - name: root_worker_inputs type: Dict[str, Any] default: null - outputs: - - default: '' + outputs: [] lineno: 56 + has_varargs: false + has_kwargs: false decorator: name: decorator doc: '' parameters: - name: handler - outputs: - - default: '' + outputs: [] lineno: 68 + has_varargs: false + has_kwargs: false wrapper: name: wrapper doc: '' parameters: [] - outputs: - - default: '' + outputs: [] lineno: 73 + has_varargs: false + has_kwargs: true translate: name: translate doc: 'Translate text files using a transformer model from Huggingface''s hub @@ -112,8 +118,10 @@ spec: default: false outputs: - doc: 'A tuple of:' - default: '' + type: Tuple[str, pd.DataFrame, dict] lineno: 135 + has_varargs: false + has_kwargs: false description: Translate text files from one language to another default_handler: translate disable_auto_mount: false diff --git a/translate/item.yaml b/translate/item.yaml index f85a55990..e63947349 100644 --- a/translate/item.yaml +++ b/translate/item.yaml @@ -1,6 +1,7 @@ apiVersion: v1 categories: - data-preparation +- huggingface - machine-learning - deep-learning - NLP @@ -28,5 +29,5 @@ spec: - torch - tqdm url: '' -version: 0.0.2 +version: 0.1.0 test_valid: True