From ec2a1b7d74a9303d6d62e40346d04167b7f1e5c9 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger Date: Thu, 12 Nov 2020 10:22:46 -0500 Subject: [PATCH 1/2] Model sharing doc --- docs/source/model_doc/marian.rst | 16 +++++++--- docs/source/model_sharing.rst | 54 +++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/docs/source/model_doc/marian.rst b/docs/source/model_doc/marian.rst index e90123adc4be..58d9f4c0a41a 100644 --- a/docs/source/model_doc/marian.rst +++ b/docs/source/model_doc/marian.rst @@ -139,11 +139,19 @@ Example of translating english to many romance languages, using old-style 2 char .. code-block::python from transformers import MarianMTModel, MarianTokenizer - src_text = [ '>>fr<< this is a sentence in english that we want to translate to french', '>>pt<< This should go to portuguese', '>>es<< And this to Spanish'] + src_text = [ + '>>fr<< this is a sentence in english that we want to translate to french', + '>>pt<< This should go to portuguese', + '>>es<< And this to Spanish' + ] + + model_name = 'Helsinki-NLP/opus-mt-en-ROMANCE' + tokenizer = MarianTokenizer.from_pretrained(model_name) + print(tokenizer.supported_language_codes) - model_name = 'Helsinki-NLP/opus-mt-en-ROMANCE' tokenizer = MarianTokenizer.from_pretrained(model_name) - print(tokenizer.supported_language_codes) model = MarianMTModel.from_pretrained(model_name) translated = - model.generate(**tokenizer.prepare_seq2seq_batch(src_text)) tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated] + model = MarianMTModel.from_pretrained(model_name) + translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text)) + tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated] # ["c'est une phrase en anglais que nous voulons traduire en français", 'Isto deve ir para o português.', 'Y esto al español'] diff --git a/docs/source/model_sharing.rst b/docs/source/model_sharing.rst index 920fabd87105..a6d3ded39d03 100644 --- a/docs/source/model_sharing.rst +++ b/docs/source/model_sharing.rst @@ -52,21 +52,21 @@ users to clone it and you (and your organization members) to push to it. First, Go in a terminal and run the following command. It should be in the virtual environment where you installed 🤗 Transformers, since that command :obj:`transformers-cli` comes from the library. -.. code-block:: +.. code-block:: bash transformers-cli login Once you are logged in with your model hub credentials, you can start building your repositories. To create a repo: -.. code-block:: +.. code-block:: bash transformers-cli repo create your-model-name This creates a repo on the model hub, which can be cloned. You can then add/remove from that repo as you would with any other git repo. -.. code-block:: +.. code-block:: bash git clone https://huggingface.co/username/your-model-name @@ -159,24 +159,25 @@ Or, if you're using the Trainer API .. code-block:: >>> trainer.save_model("path/to/awesome-name-you-picked") + >>> tokenizer.save_pretrained("path/to/repo/clone/your-model-name") You can then add these files to the staging environment and verify that they have been correctly staged with the ``git status`` command: -.. code-block:: +.. code-block:: bash git add --all git status Finally, the files should be comitted: -.. code-block:: +.. code-block:: bash git commit -m "First version of the your-model-name model and tokenizer." And pushed to the remote: -.. code-block:: +.. code-block:: bash git push @@ -199,7 +200,7 @@ don't forget to link to its model card so that people can fully trace how your m If you have never made a pull request to the 🤗 Transformers repo, look at the :doc:`contributing guide ` to see the steps to follow. -.. Note:: +.. note:: You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file inside `path/to/awesome-name-you-picked/`. @@ -225,3 +226,42 @@ You may specify a revision by using the ``revision`` flag in the ``from_pretrain >>> "julien-c/EsperBERTo-small", >>> revision="v2.0.1" # tag name, or branch name, or commit hash >>> ) + +Workflow in a Colab notebook +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're in a Colab notebook (or similar) with no direct access to a terminal, here is the workflow you can use to +upload your model. You can execute each one of them in a cell by adding a ! at the beginning. + +First you need to install `git-lfs` in the environment used by the notebook: + +.. code-block:: bash + + sudo apt-get install git-lfs + +Then you can use the :obj:`transformers-cli` to create your new repo: + + +.. code-block:: bash + + transformers-cli login + transformers-cli repo create your-model-name + +Once it's created, you can clone it and configure it (replace username by your username on huggingface.co): + +.. code-block:: bash + + git clone https://huggingface.co/username/your-model-name + cd your-model-name + git lfs install + git config --global user.email "email@example.com" + +Once you've saved your model inside, you can add it and push it with usual git commands. Note that you have to replace +`username:password` with your username and password to huggingface.co. + +.. code-block:: bash + + git add . + git commit -m "Initial commit" + git push https://username:password@huggingface.co/username/your-model-name + \ No newline at end of file From c4ce6c365d32948f45a80ae6b3f9190e0c683a72 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger Date: Thu, 12 Nov 2020 10:31:03 -0500 Subject: [PATCH 2/2] Style --- docs/source/model_sharing.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/model_sharing.rst b/docs/source/model_sharing.rst index a6d3ded39d03..81f9326163b5 100644 --- a/docs/source/model_sharing.rst +++ b/docs/source/model_sharing.rst @@ -264,4 +264,3 @@ Once you've saved your model inside, you can add it and push it with usual git c git add . git commit -m "Initial commit" git push https://username:password@huggingface.co/username/your-model-name - \ No newline at end of file