diff options
| -rwxr-xr-x | contrib/oauth2.py | 348 | ||||
| -rw-r--r-- | docs/meli.conf.5 | 43 | ||||
| -rw-r--r-- | melib/src/backends/imap.rs | 26 | ||||
| -rw-r--r-- | melib/src/backends/imap/connection.rs | 44 |
4 files changed, 452 insertions, 9 deletions
diff --git a/contrib/oauth2.py b/contrib/oauth2.py new file mode 100755 index 00000000..a1ab65f2 --- /dev/null +++ b/contrib/oauth2.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# +# Copyright 2012 Google Inc. +# Copyright 2020 Manos Pitsidianakis +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# + # http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Performs client tasks for testing IMAP OAuth2 authentication. + +To use this script, you'll need to have registered with Google as an OAuth +application and obtained an OAuth client ID and client secret. +See https://developers.google.com/identity/protocols/OAuth2 for instructions on +registering and for documentation of the APIs invoked by this code. + +This script has 3 modes of operation. + +1. The first mode is used to generate and authorize an OAuth2 token, the +first step in logging in via OAuth2. + + oauth2 --user=xxx@gmail.com \ + --client_id=1038[...].apps.googleusercontent.com \ + --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \ + --generate_oauth2_token + +The script will converse with Google and generate an oauth request +token, then present you with a URL you should visit in your browser to +authorize the token. Once you get the verification code from the Google +website, enter it into the script to get your OAuth access token. The output +from this command will contain the access token, a refresh token, and some +metadata about the tokens. The access token can be used until it expires, and +the refresh token lasts indefinitely, so you should record these values for +reuse. + +2. The script will generate new access tokens using a refresh token. + + oauth2 --user=xxx@gmail.com \ + --client_id=1038[...].apps.googleusercontent.com \ + --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \ + --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA + +3. The script will generate an OAuth2 string that can be fed +directly to IMAP or SMTP. This is triggered with the --generate_oauth2_string +option. + + oauth2 --generate_oauth2_string --user=xxx@gmail.com \ + --access_token=ya29.AGy[...]ezLg + +The output of this mode will be a base64-encoded string. To use it, connect to a +IMAPFE and pass it as the second argument to the AUTHENTICATE command. + + a AUTHENTICATE XOAUTH2 a9sha9sfs[...]9dfja929dk== +""" + +import base64 +import imaplib +import json +from optparse import OptionParser +import smtplib +import sys +import urllib.request, urllib.parse, urllib.error + + +def SetupOptionParser(): + # Usage message is the module's docstring. + parser = OptionParser(usage=__doc__) + parser.add_option('--generate_oauth2_token', + action='store_true', + dest='generate_oauth2_token', + help='generates an OAuth2 token for testing') + parser.add_option('--generate_oauth2_string', + action='store_true', + dest='generate_oauth2_string', + help='generates an initial client response string for ' + 'OAuth2') + parser.add_option('--client_id', + default=None, + help='Client ID of the application that is authenticating. ' + 'See OAuth2 documentation for details.') + parser.add_option('--client_secret', + default=None, + help='Client secret of the application that is ' + 'authenticating. See OAuth2 documentation for ' + 'details.') + parser.add_option('--access_token', + default=None, + help='OAuth2 access token') + parser.add_option('--refresh_token', + default=None, + help='OAuth2 refresh token') + parser.add_option('--scope', + default='https://mail.google.com/', + help='scope for the access token. Multiple scopes can be ' + 'listed separated by spaces with the whole argument ' + 'quoted.') + parser.add_option('--test_imap_authentication', + action='store_true', + dest='test_imap_authentication', + help='attempts to authenticate to IMAP') + parser.add_option('--test_smtp_authentication', + action='store_true', + dest='test_smtp_authentication', + help='attempts to authenticate to SMTP') + parser.add_option('--user', + default=None, + help='email address of user whose account is being ' + 'accessed') + parser.add_option('--quiet', + action='store_true', + default=False, + dest='quiet', + help='Omit verbose descriptions and only print ' + 'machine-readable outputs.') + return parser + + +# The URL root for accessing Google Accounts. +GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com' + + +# Hardcoded dummy redirect URI for non-web apps. +REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob' + + +def AccountsUrl(command): + """Generates the Google Accounts URL. + + Args: + command: The command to execute. + + Returns: + A URL for the given command. + """ + return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command) + + +def UrlEscape(text): + # See OAUTH 5.1 for a definition of which characters need to be escaped. + return urllib.parse.quote(text, safe='~-._') + + +def UrlUnescape(text): + # See OAUTH 5.1 for a definition of which characters need to be escaped. + return urllib.parse.unquote(text) + + +def FormatUrlParams(params): + """Formats parameters into a URL query string. + + Args: + params: A key-value map. + + Returns: + A URL query string version of the given parameters. + """ + param_fragments = [] + for param in sorted(iter(params.items()), key=lambda x: x[0]): + param_fragments.append('%s=%s' % (param[0], UrlEscape(param[1]))) + return '&'.join(param_fragments) + + +def GeneratePermissionUrl(client_id, scope='https://mail.google.com/'): + """Generates the URL for authorizing access. + + This uses the "OAuth2 for Installed Applications" flow described at + https://developers.google.com/accounts/docs/OAuth2InstalledApp + + Args: + client_id: Client ID obtained by registering your app. + scope: scope for access token, e.g. 'https://mail.google.com' + Returns: + A URL that the user should visit in their browser. + """ + params = {} + params['client_id'] = client_id + params['redirect_uri'] = REDIRECT_URI + params['scope'] = scope + params['response_type'] = 'code' + return '%s?%s' % (AccountsUrl('o/oauth2/auth'), + FormatUrlParams(params)) + + +def AuthorizeTokens(client_id, client_secret, authorization_code): + """Obtains OAuth access token and refresh token. + + This uses the application portion of the "OAuth2 for Installed Applications" + flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse + + Args: + client_id: Client ID obtained by registering your app. + client_secret: Client secret obtained by registering your app. + authorization_code: code generated by Google Accounts after user grants + permission. + Returns: + The decoded response from the Google Accounts server, as a dict. Expected + fields include 'access_token', 'expires_in', and 'refresh_token'. + """ + params = {} + params['client_id'] = client_id + params['client_secret'] = client_secret + params['code'] = authorization_code + params['redirect_uri'] = REDIRECT_URI + params['grant_type'] = 'authorization_code' + request_url = AccountsUrl('o/oauth2/token') + + response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read() + return json.loads(response) + + +def RefreshToken(client_id, client_secret, refresh_token): + """Obtains a new token given a refresh token. + + See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh + + Args: + client_id: Client ID obtained by registering your app. + client_secret: Client secret obtained by registering your app. + refresh_token: A previously-obtained refresh token. + Returns: + The decoded response from the Google Accounts server, as a dict. Expected + fields include 'access_token', 'expires_in', and 'refresh_token'. + """ + params = {} + params['client_id'] = client_id + params['client_secret'] = client_secret + params['refresh_token'] = refresh_token + params['grant_type'] = 'refresh_token' + request_url = AccountsUrl('o/oauth2/token') + + response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read() + return json.loads(response) + + +def GenerateOAuth2String(username, access_token, base64_encode=True): + """Generates an IMAP OAuth2 authentication string. + + See https://developers.google.com/google-apps/gmail/oauth2_overview + + Args: + username: the username (email address) of the account to authenticate + access_token: An OAuth2 access token. + base64_encode: Whether to base64-encode the output. + + Returns: + The SASL argument for the OAuth2 mechanism. + """ + auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token) + if base64_encode: + auth_string = base64.b64encode(bytes(auth_string, 'utf-8')) + return auth_string + + +def TestImapAuthentication(user, auth_string): + """Authenticates to IMAP with the given auth_string. + + Prints a debug trace of the attempted IMAP connection. + + Args: + user: The Gmail username (full email address) + auth_string: A valid OAuth2 string, as returned by GenerateOAuth2String. + Must not be base64-encoded, since imaplib does its own base64-encoding. + """ + print() + imap_conn = imaplib.IMAP4_SSL('imap.gmail.com') + imap_conn.debug = 4 + imap_conn.authenticate('XOAUTH2', lambda x: auth_string) + imap_conn.select('INBOX') + + +def TestSmtpAuthentication(user, auth_string): + """Authenticates to SMTP with the given auth_string. + + Args: + user: The Gmail username (full email address) + auth_string: A valid OAuth2 string, not base64-encoded, as returned by + GenerateOAuth2String. + """ + print() + smtp_conn = smtplib.SMTP('smtp.gmail.com', 587) + smtp_conn.set_debuglevel(True) + smtp_conn.ehlo('test') + smtp_conn.starttls() + smtp_conn.docmd('AUTH', 'XOAUTH2 ' + base64.b64encode(auth_string)) + + +def RequireOptions(options, *args): + missing = [arg for arg in args if getattr(options, arg) is None] + if missing: + print('Missing options: %s' % ' '.join(missing), file=sys.stderr) + sys.exit(-1) + + +def main(argv): + options_parser = SetupOptionParser() + (options, args) = options_parser.parse_args() + if options.refresh_token: + RequireOptions(options, 'client_id', 'client_secret') + response = RefreshToken(options.client_id, options.client_secret, + options.refresh_token) + if options.quiet: + print(response['access_token']) + else: + print('Access Token: %s' % response['access_token']) + print('Access Token Expiration Seconds: %s' % response['expires_in']) + elif options.generate_oauth2_string: + RequireOptions(options, 'user', 'access_token') + oauth2_string = GenerateOAuth2String(options.user, options.access_token) + if options.quiet: + print(oauth2_string.decode('utf-8')) + else: + print('OAuth2 argument:\n' + oauth2_string.decode('utf-8')) + elif options.generate_oauth2_token: + RequireOptions(options, 'client_id', 'client_secret') + print('To authorize token, visit this url and follow the directions:') + print(' %s' % GeneratePermissionUrl(options.client_id, options.scope)) + authorization_code = input('Enter verification code: ') + response = AuthorizeTokens(options.client_id, options.client_secret, + authorization_code) + print('Refresh Token: %s' % response['refresh_token']) + print('Access Token: %s' % response['access_token']) + print('Access Token Expiration Seconds: %s' % response['expires_in']) + elif options.test_imap_authentication: + RequireOptions(options, 'user', 'access_token') + TestImapAuthentication(options.user, + GenerateOAuth2String(options.user, options.access_token, + base64_encode=False)) + elif options.test_smtp_authentication: + RequireOptions(options, 'user', 'access_token') + TestSmtpAuthentication(options.user, + GenerateOAuth2String(options.user, options.access_token, + base64_encode=False)) + else: + options_parser.print_help() + print('Nothing to do, exiting.') + return + + +if __name__ == '__main__': + main(sys.argv) diff --git a/docs/meli.conf.5 b/docs/meli.conf.5 index cc6e3c13..1d306287 100644 --- a/docs/meli.conf.5 +++ b/docs/meli.conf.5 @@ -235,11 +235,25 @@ Do not validate TLS certificates. Use IDLE extension. .\" default value .Pq Em true +.It Ic use_condstore Ar boolean +.Pq Em optional +Use CONDSTORE extension. +.\" default value +.Pq Em true .It Ic use_deflate Ar boolean .Pq Em optional Use COMPRESS=DEFLATE extension (if built with DEFLATE support). .\" default value .Pq Em true +.It Ic use_oauth2 Ar boolean +.Pq Em optional +Use OAUTH2 authentication. +Can only be used with +.Ic server_password_command +which should return a base64-encoded OAUTH2 token ready to be passed to IMAP. +For help on setup with Gmail, see Gmail section below. +.\" default value +.Pq Em false .It Ic timeout Ar integer .Pq Em optional Timeout to use for server connections in seconds. @@ -247,6 +261,35 @@ A timeout of 0 seconds means there's no timeout. .\" default value .Pq Em 16 .El +.Ss Gmail +Gmail has non-standard IMAP behaviors that need to be worked around. +.Ss Gmail - sending mail +Option +.Ic store_sent_mail +should be disabled since Gmail auto-saves sent mail by its own. +.Ss Gmail OAUTH2 +To use OAUTH2, you must go through a process to register your own private "application" with Google that can use OAUTH2 tokens. +For convenience in the meli repository under the +.Pa contrib/ +directory you can find a python3 file named oauth2.py to generate and request the appropriate data to perform OAUTH2 authentication. +Steps: +.Bl -bullet -compact +.It +In Google APIs, create a custom OAuth client ID and note down the Client ID and Client Secret. +You may need to create a consent screen; follow the steps described in the website. +.It +Run the oauth2.py script as follows (after adjusting binary paths and credentials): +.Cm python3 oauth2.py --user=xxx@gmail.com --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --generate_oauth2_token +and follow the instructions. +Note down the refresh token. +.It +In +.Ic server_password_command +enter a command like this (after adjusting binary paths and credentials): +.Cm TOKEN=$(python3 oauth2.py --user=xxx@gmail.com --quiet --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA) && python3 oauth2.py --user=xxx@gmail.com --generate_oauth2_string --quiet --access_token=$TOKEN +.It +On startup, meli should evaluate this command which if successful must only return a base64-encoded token ready to be passed to IMAP. +.El .Ss JMAP only JMAP specific options .Bl -tag -width 36n diff --git a/melib/src/backends/imap.rs b/melib/src/backends/imap.rs index a060ca97..e3186a65 100644 --- a/melib/src/backends/imap.rs +++ b/melib/src/backends/imap.rs @@ -64,6 +64,7 @@ pub type UIDVALIDITY = UID; pub type MessageSequenceNumber = ImapNum; pub static SUPPORTED_CAPABILITIES: &[&str] = &[ + "AUTH=OAUTH2", #[cfg(feature = "deflate_compression")] "COMPRESS=DEFLATE", "CONDSTORE", @@ -232,6 +233,7 @@ impl MailBackend for ImapType { #[cfg(feature = "deflate_compression")] deflate, condstore, + oauth2, }, } = self.server_conf.protocol { @@ -273,6 +275,15 @@ impl MailBackend for ImapType { }; } } + "AUTH=OAUTH2" => { + if oauth2 { + *status = MailBackendExtensionStatus::Enabled { comment: None }; + } else { + *status = MailBackendExtensionStatus::Supported { + comment: Some("Disabled by user configuration"), + }; + } + } _ => { if SUPPORTED_CAPABILITIES .iter() @@ -1218,7 +1229,14 @@ impl ImapType { ) -> Result<Box<dyn MailBackend>> { let server_hostname = get_conf_val!(s["server_hostname"])?; let server_username = get_conf_val!(s["server_username"])?; + let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?; let server_password = if !s.extra.contains_key("server_password_command") { + if use_oauth2 { + return Err(MeliError::new(format!( + "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.", + s.name, + ))); + } get_conf_val!(s["server_password"])?.to_string() } else { let invocation = get_conf_val!(s["server_password_command"])?; @@ -1275,6 +1293,7 @@ impl ImapType { condstore: get_conf_val!(s["use_condstore"], true)?, #[cfg(feature = "deflate_compression")] deflate: get_conf_val!(s["use_deflate"], true)?, + oauth2: use_oauth2, }, }, timeout, @@ -1463,7 +1482,14 @@ impl ImapType { pub fn validate_config(s: &AccountSettings) -> Result<()> { get_conf_val!(s["server_hostname"])?; get_conf_val!(s["server_username"])?; + let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?; if !s.extra.contains_key("server_password_command") { + if use_oauth2 { + return Err(MeliError::new(format!( + "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.", + s.name, + ))); + } get_conf_val!(s["server_password"])?; } else if s.extra.contains_key("server_password") { return Err(MeliError::new(format!( diff --git a/melib/src/backends/imap/connection.rs b/melib/src/backends/imap/connection.rs index 69aa622a..1d2353c1 100644 --- a/melib/src/backends/imap/connection.rs +++ b/melib/src/backends/imap/connection.rs @@ -63,6 +63,7 @@ pub struct ImapExtensionUse { pub idle: bool, #[cfg(feature = "deflate_compression")] pub deflate: bool, + pub oauth2: bool, } impl Default for ImapExtensionUse { @@ -72,6 +73,7 @@ impl Default for ImapExtensionUse { idle: true, #[cfg(feature = "deflate_compression")] deflate: true, + oauth2: false, } } } @@ -351,16 +353,39 @@ impl ImapStream { .set_err_kind(crate::error::ErrorKind::Authentication)); } - let mut capabilities = None; - ret.send_command( - format!( - "LOGIN \"{}\" \"{}\"", - &server_conf.server_username, &server_conf.server_password - ) - .as_bytes(), - ) - .await?; + match server_conf.protocol { + ImapProtocol::IMAP { + extension_use: ImapExtensionUse { oauth2, .. }, + } if oauth2 => { + if !capabilities + .iter() + .any(|cap| cap.eq_ignore_ascii_case(b"AUTH=XOAUTH2")) + { + return Err(MeliError::new(format!( + "Could not connect to {}: OAUTH2 is enabled but server did not return AUTH=XOAUTH2 capability. Returned capabilities were: {}", + &server_conf.server_hostname, + capabilities.iter().map(|capability| + String::from_utf8_lossy(capability).to_string()).collect::<Vec<String>>().join(" ") + ))); + } + ret.send_command( + format!("AUTHENTICATE XOAUTH2 {}", &server_conf.server_password).as_bytes(), + ) + .await?; + } + _ => { + ret.send_command( + format!( + "LOGIN \"{}\" \"{}\"", + &server_conf.server_username, &server_conf.server_password + ) + .as_bytes(), + ) + .await?; + } + } let tag_start = format!("M{} ", (ret.cmd_id - 1)); + let mut capabilities = None; loop { ret.read_lines(&mut res, &[], false).await?; @@ -604,6 +629,7 @@ impl ImapConnection { #[cfg(feature = "deflate_compression")] deflate, idle: _idle, + oauth2: _, }, } => { if capabilities.contains(&b"CONDSTORE"[..]) && condstore { |