Skip to content

Add support for public repos on travis-ci.com #310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
asmeurer merged 33 commits into from
Nov 30, 2018
Merged

Add support for public repos on travis-ci.com #310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
31defe9
Fix the question mark location for one of the doctr configure questions
asmeurer Aug 6, 2018
c0bbd93
Make check_repo_exists return whether or not the repo is on travis-ci…
asmeurer Aug 6, 2018
4fff488
Clarify docstring
asmeurer Aug 6, 2018
7ed3640
Change is_private argument to encrypt_variable to dotcom
asmeurer Aug 6, 2018
de6f84f
Better separation of travis.com and travis.org in check_repo_exists()
asmeurer Aug 7, 2018
f2fc924
Fix check_repo_exists() tests
asmeurer Aug 7, 2018
3e0b51e
Fix service name to match the actual URL
asmeurer Aug 7, 2018
3b0033e
Fix is_private logic in __main__.py
asmeurer Aug 7, 2018
17c159c
Fix encrypt_variable for both travis-ci.org and .com
asmeurer Aug 7, 2018
6fcd844
Add ask flag to check_repo_exists
asmeurer Aug 7, 2018
be03af3
Fix error message
asmeurer Aug 7, 2018
486527c
Fix check_repo_exists() tests
asmeurer Aug 7, 2018
564c69d
Add comment about check_repo_exists test
asmeurer Aug 7, 2018
d99639e
Print a note about the personal access token
asmeurer Aug 8, 2018
2a12448
Fix getting of the public key on travis-ci.com
asmeurer Aug 27, 2018
f917ae5
Add error handling if the public key cannot be found on travis-ci.com
asmeurer Aug 27, 2018
c722773
Delete the GitHub personal access token in a finally block
asmeurer Aug 27, 2018
b6a391e
Move try statement up
asmeurer Aug 27, 2018
127383d
Print RuntimeError error messages in red
asmeurer Aug 27, 2018
6f7f041
Merge branch 'master' into travis-ci-dot-com
asmeurer Aug 27, 2018
51be924
Add a function to determine if doctr is running on travis-ci.com or t…
asmeurer Aug 28, 2018
f1d4080
Add test print to test travis_tld
asmeurer Aug 28, 2018
d40c545
Fix debug print
asmeurer Aug 28, 2018
26ad452
Some more debug prints
asmeurer Aug 28, 2018
0642817
Merge branch 'master' into travis-ci-dot-com
asmeurer Nov 27, 2018
dc68e51
Use TRAVIS_JOB_WEB_URL to get the URL for the commit message
asmeurer Nov 27, 2018
63f3669
Fix travis-ci.com being the default choice in configure
asmeurer Nov 27, 2018
648a7de
Add color guide to the common.py file
asmeurer Nov 27, 2018
7cd59f6
Add note that GitHub will email you about th personal access token
asmeurer Nov 27, 2018
f4e3a7a
Add missing import
asmeurer Nov 28, 2018
a9e8b79
Add --no-authenticate flag to doctr configure
asmeurer Nov 28, 2018
4da7d99
Print an error message when --no-authenticate us used with a private …
asmeurer Nov 28, 2018
a6b8cfc
Merge branch 'master' into travis-ci-dot-com
asmeurer Nov 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions doctr/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,9 +196,14 @@ def get_parser(config=None):
public repositories for the user. This option is not recommended
unless you are using a separate GitHub user for deploying.""")
configure_parser.add_argument("--no-upload-key", action="store_false", default=True,
dest="upload_key", help="""Don't automatically upload the deploy key to GitHub. If you select this
option, you will not be prompted for your GitHub credentials, so this option is not compatible with
private repositories.""")
dest="upload_key", help="""Don't automatically upload the deploy key to GitHub. To prevent doctr
configure from asking for your GitHub credentials, use
--no-authenticate.""")
configure_parser.add_argument("--no-authenticate", action="store_false",
default=True, dest="authenticate", help="""Don't authenticate with GitHub. This option implies --no-upload-key. Note:
it is not possible to configure travis-ci.com with this option, only
.org (see https://github.com/travis-ci/travis-ci/issues/9954). This
option is also not compatible with private repositories.""")
configure_parser.add_argument('--key-path', default=None,
help="""Path to save the encrypted GitHub deploy key. The default is github_deploy_key_+
deploy respository name. The .enc extension is added to the file automatically.""")
Expand Down Expand Up @@ -372,6 +377,8 @@ def configure(args, parser):
parser.error(red("doctr appears to be running on Travis. Use "
"doctr configure --force to run anyway."))

if not args.authenticate:
args.upload_key = False

print(green(dedent("""\
Welcome to Doctr.
Expand All @@ -381,7 +388,8 @@ def configure(args, parser):
""")))

login_kwargs = {}
if args.upload_key:

if args.authenticate:
while not login_kwargs:
try:
login_kwargs = GitHub_login()
Expand All @@ -395,13 +403,20 @@ def configure(args, parser):
while not get_build_repo:
try:
if default_repo:
build_repo = input("What repo do you want to build the docs for [{default_repo}]? ".format(default_repo=blue(default_repo)))
build_repo = input("What repo do you want to build the docs for? [{default_repo}] ".format(default_repo=blue(default_repo)))
if not build_repo:
build_repo = default_repo
else:
build_repo = input("What repo do you want to build the docs for (org/reponame, like 'drdoctr/doctr')? ")
is_private = check_repo_exists(build_repo, service='github', **login_kwargs)
check_repo_exists(build_repo, service='travis')
is_private = check_repo_exists(build_repo, service='github',
**login_kwargs)
if is_private and not args.authenticate:
sys.exit(red("--no-authenticate is not supported for private repositories."))

is_private = check_repo_exists(build_repo, service='travis', ask=True) or is_private
if is_private and not args.authenticate:
sys.exit(red("--no-authenticate is not supported for travis-ci.com. See https://github.com/travis-ci/travis-ci/issues/9954."))

get_build_repo = True
except GitHubError:
raise
Expand Down
9 changes: 9 additions & 0 deletions doctr/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,15 @@
Code used for both Travis and local (deploy and configure)
"""

# Color guide
#
# - red: Error and warning messages
# - green: Welcome messages (use sparingly)
# - blue: Default values
# - bold_magenta: Action items
# - bold_black: Parts of code to be run or copied that should be modified


def red(text):
return "\033[31m%s\033[0m" % text

Expand Down
140 changes: 103 additions & 37 deletions doctr/local.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from cryptography.hazmat.primitives import serialization


from .common import red
from .common import red, blue, green

def encrypt_variable(variable, build_repo, *, public_key=None, is_private=False, **login_kwargs):
"""
Expand All @@ -34,38 +34,59 @@ def encrypt_variable(variable, build_repo, *, public_key=None, is_private=False,

``public_key`` should be a pem format public key, obtained from Travis if
not provided.

``dotcom`` should be True if the service is travis-ci.com and False if it
is travis-ci.org. Not that travis-ci.com requires creating a temporary
authentication on GitHub, which is deleted automatically (regardless of
whether or not the repo is private).
"""
if not isinstance(variable, bytes):
raise TypeError("variable should be bytes")

if not b"=" in variable:
raise ValueError("variable should be of the form 'VARIABLE=value'")

APIv2 = {'Accept': 'application/vnd.travis-ci.2+json'}
APIv3 = {"Travis-API-Version": "3"}
if not public_key:
headers = {'Accept': 'application/vnd.travis-ci.2+json',
'Content-Type': 'application/json',
'User-Agent': 'MyClient/1.0.0'}
if is_private:
tok_dict = generate_GitHub_token(scopes=["read:org", "user:email", "repo"],
note="temporary token to auth against travis",
**login_kwargs)
data = {'github_token': tok_dict['token']}
token_id = tok_dict['id']
res = requests.post('https://api.travis-ci.com/auth/github', data=json.dumps(data), headers=headers)
_headers = {
'Content-Type': 'application/json',
'User-Agent': 'MyClient/1.0.0',
}
headersv2 = {**_headers, **APIv2}
headersv3 = {**_headers, **APIv3}
token_id = None
try:
if is_private:
print(green("I need to generate a temporary token with GitHub to authenticate with Travis. You may get a warning email from GitHub about this."))
print(green("It will be deleted immediately. If you still see it after this at https://github.com/settings/tokens after please delete it manually."))
# /auth/github doesn't seem to exist in the Travis API v3.
tok_dict = generate_GitHub_token(scopes=["read:org", "user:email", "repo"],
note="temporary token for doctr to auth against travis (delete me)",
**login_kwargs)
data = {'github_token': tok_dict['token']}
token_id = tok_dict['id']
res = requests.post('https://api.travis-ci.com/auth/github', data=json.dumps(data), headers=headersv2)
res.raise_for_status()
headersv3['Authorization'] = 'token {}'.format(res.json()['access_token'])
res = requests.get('https://api.travis-ci.com/repo/{build_repo}/key_pair/generated'.format(build_repo=urllib.parse.quote(build_repo,
safe='')), headers=headersv3)
if res.json().get('file') == 'not found':
print(headersv3)
raise RuntimeError("Could not find the Travis public key for %s" % build_repo)
public_key = res.json()['public_key']
else:
res = requests.get('https://api.travis-ci.org/repos/{build_repo}/key'.format(build_repo=build_repo), headers=headersv2)
public_key = res.json()['key']

if res.status_code == requests.codes.not_found:
raise RuntimeError('Could not find requested repo on Travis. Is Travis enabled?')
res.raise_for_status()
headers['Authorization'] = 'token {}'.format(res.json()['access_token'])
tld = 'com'
else:
tld = 'org'
res = requests.get('https://api.travis-ci.{tld}/repos/{build_repo}/key'.format(build_repo=build_repo, tld=tld),
headers=headers)
if res.status_code == requests.codes.not_found:
raise RuntimeError('Could not find requested repo on Travis. Is Travis enabled?')
res.raise_for_status()
public_key = res.json()['key']
# Remove temporary GH token
if is_private:
delete_GitHub_token(token_id, **login_kwargs)

finally:
# Remove temporary GH token
if is_private and token_id:
delete_GitHub_token(token_id, **login_kwargs)

public_key = public_key.replace("RSA PUBLIC KEY", "PUBLIC KEY").encode('utf-8')
key = serialization.load_pem_public_key(public_key, backend=default_backend())
Expand Down Expand Up @@ -285,7 +306,8 @@ def generate_ssh_key():

return private_key, public_key

def check_repo_exists(deploy_repo, service='github', *, auth=None, headers=None):
def check_repo_exists(deploy_repo, service='github', *, auth=None,
headers=None, ask=False):
"""
Checks that the repository exists on GitHub.

Expand All @@ -294,7 +316,16 @@ def check_repo_exists(deploy_repo, service='github', *, auth=None, headers=None)

Raises ``RuntimeError`` if the repo is not valid.

Returns whether or not the repo is private
Returns whether or not the repo requires authorization to access. Private
repos require authorization, as to repos on travis-ci.com, regardless of
whether or not it is private.

For service='travis', if ask=True, it will ask at the command line if both
travis-ci.org and travis-ci.com exist. If ask=False, service='travis' will
check travis-ci.com first and only check travis-ci.org if it doesn't
exist. ask=True does nothing for service='github',
service='travis-ci.com', service='travis-ci.org'.

"""
headers = headers or {}
if deploy_repo.count("/") != 1:
Expand All @@ -303,30 +334,65 @@ def check_repo_exists(deploy_repo, service='github', *, auth=None, headers=None)
user, repo = deploy_repo.split('/')
if service == 'github':
REPO_URL = 'https://api.github.com/repos/{user}/{repo}'
elif service == 'travis':
elif service == 'travis' or service == 'travis-ci.com':
REPO_URL = 'https://api.travis-ci.com/repo/{user}%2F{repo}'
headers['Travis-API-Version'] = '3'
elif service == 'travis-ci.org':
REPO_URL = 'https://api.travis-ci.org/repo/{user}%2F{repo}'
headers['Travis-API-Version'] = '3'
else:
raise RuntimeError('Invalid service specified for repo check (neither "travis" nor "github")')
raise RuntimeError('Invalid service specified for repo check (should be one of {"github", "travis", "travis-ci.com", "travis-ci.org"}')

wiki = False
if repo.endswith('.wiki') and service == 'github':
wiki = True
repo = repo[:-5]

r = requests.get(REPO_URL.format(user=urllib.parse.quote(user),
repo=urllib.parse.quote(repo)), auth=auth, headers=headers)
def _try(url):
r = requests.get(url, auth=auth, headers=headers)

if r.status_code == requests.codes.not_found:
return False
if service == 'github':
GitHub_raise_for_status(r)
else:
r.raise_for_status()
return r

r = _try(REPO_URL.format(user=urllib.parse.quote(user),
repo=urllib.parse.quote(repo)))

if r.status_code == requests.codes.not_found:
if service == 'travis':
REPO_URL = 'https://api.travis-ci.org/repo/{user}%2F{repo}'
r_org = _try(REPO_URL.format(user=urllib.parse.quote(user),
repo=urllib.parse.quote(repo)))
if not r:
r = r_org
else:
if r and r_org:
if ask:
while True:
print("{user}/{repo} appears to exist on both travis-ci.org and travis-ci.com.".format(user=user, repo=repo))
preferred = input("Which do you want to use? [{default}/travis-ci.org] ".format(default=blue("travis-ci.com")))
preferred = preferred.lower().strip()
if preferred in ['o', 'org', '.org', 'travis-ci.org']:
r = r_org
service = 'travis-ci.org'
break
elif preferred in ['c', 'com', '.com', 'travis-ci.com', '']:
service = 'travis-ci.com'
break
else:
print(red("Please type 'travis-ci.com' or 'travis-ci.org'."))
else:
service = 'travis-ci.com'


if not r:
raise RuntimeError('"{user}/{repo}" not found on {service}'.format(user=user,
repo=repo,
service=service))

if service == 'github':
GitHub_raise_for_status(r)
else:
r.raise_for_status()

private = r.json().get('private', False)

if wiki and not private:
Expand All @@ -337,7 +403,7 @@ def check_repo_exists(deploy_repo, service='github', *, auth=None, headers=None)
raise RuntimeError('Wiki not found. Please create a wiki')
return False

return private
return private or (service == 'travis-ci.com')

GIT_URL = re.compile(r'(?:git@|https://|git://)github\.com[:/](.*?)(?:\.git)?')

Expand Down
6 changes: 5 additions & 1 deletion doctr/tests/test_local.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,12 @@ def test_travis_bad_repo():
# Travis is case-sensitive
check_repo_exists('drdoctr/DoCtR', service='travis')

# This test may need to be adjusted as travis-ci.org gets merged into
# travis-ci.com. Currently drdoctr/doctr is enabled on both.
def test_travis_repo_exists():
assert not check_repo_exists('drdoctr/doctr', service='travis')
assert not check_repo_exists('drdoctr/doctr', service='travis-ci.org')
assert check_repo_exists('drdoctr/doctr', service='travis-ci.com')
assert check_repo_exists('drdoctr/doctr', service='travis')

def test_GIT_URL():
for url in [
Expand Down
6 changes: 3 additions & 3 deletions doctr/travis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ def commit_docs(*, added, removed):
TRAVIS_BRANCH = os.environ.get("TRAVIS_BRANCH", "<unknown>")
TRAVIS_COMMIT = os.environ.get("TRAVIS_COMMIT", "<unknown>")
TRAVIS_REPO_SLUG = os.environ.get("TRAVIS_REPO_SLUG", "<unknown>")
TRAVIS_JOB_ID = os.environ.get("TRAVIS_JOB_ID", "")
TRAVIS_JOB_WEB_URL = os.environ.get("TRAVIS_JOB_WEB_URL", "<unknown>")
TRAVIS_TAG = os.environ.get("TRAVIS_TAG", "")
branch = "tag" if TRAVIS_TAG else "branch"

Expand All @@ -490,7 +490,7 @@ def commit_docs(*, added, removed):
{TRAVIS_COMMIT}.

The Travis build that generated this commit is at
https://travis-ci.org/{TRAVIS_REPO_SLUG}/jobs/{TRAVIS_JOB_ID}.
{TRAVIS_JOB_WEB_URL}.

The doctr command that was run is

Expand All @@ -501,7 +501,7 @@ def commit_docs(*, added, removed):
TRAVIS_BRANCH=TRAVIS_BRANCH,
TRAVIS_COMMIT=TRAVIS_COMMIT,
TRAVIS_REPO_SLUG=TRAVIS_REPO_SLUG,
TRAVIS_JOB_ID=TRAVIS_JOB_ID,
TRAVIS_JOB_WEB_URL=TRAVIS_JOB_WEB_URL,
DOCTR_COMMAND=DOCTR_COMMAND,
)

Expand Down