More unit tests for List and ListItem resources, bug fixes (#171), initial support for Federated Authentication with SharePoint Online

Author: vgrem

examples/sharepoint/web_read_direct.py

@@ -3,11 +3,11 @@
 from settings import settings
 
 from office365.runtime.auth.authentication_context import AuthenticationContext
-from office365.runtime.client_request import ClientRequest
 from office365.runtime.utilities.request_options import RequestOptions
 from office365.sharepoint.client_context import ClientContext
 
 if __name__ == '__main__':
+
     context_auth = AuthenticationContext(url=settings['url'])
     if context_auth.acquire_token_for_user(username=settings['user_credentials']['username'],
                                            password=settings['user_credentials']['password']):

office365/runtime/auth/FederatedSAML.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope" xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" xmlns:saml="urn:oasis:names:tc:SAML:1.0:assertion" xmlns:wsp="http://schemas.xmlsoap.org/ws/2004/09/policy" xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd" xmlns:wsa="http://www.w3.org/2005/08/addressing" xmlns:wssc="http://schemas.xmlsoap.org/ws/2005/02/sc" xmlns:wst="http://schemas.xmlsoap.org/ws/2005/02/trust">
+    <s:Header>
+        <wsa:Action s:mustUnderstand="1">http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</wsa:Action>
+        <wsa:To s:mustUnderstand="1">{auth_url}</wsa:To>
+        <wsa:MessageID>{message_id}</wsa:MessageID>
+        <wsse:Security>
+            <wsse:UsernameToken wsu:Id="user">
+                <wsse:Username>{username}</wsse:Username>
+                <wsse:Password>{password}</wsse:Password>
+            </wsse:UsernameToken>
+        </wsse:Security>
+    </s:Header>
+    <s:Body>
+        <wst:RequestSecurityToken Id="RST0">
+            <wst:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</wst:RequestType>
+            <wsp:AppliesTo>
+                <wsa:EndpointReference>
+                    <wsa:Address>urn:federation:MicrosoftOnline</wsa:Address>
+                </wsa:EndpointReference>
+            </wsp:AppliesTo>
+            <wst:KeyType>http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey</wst:KeyType>
+        </wst:RequestSecurityToken>
+    </s:Body>
+</s:Envelope>

office365/runtime/auth/RST2.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<S:Envelope xmlns:S="http://www.w3.org/2003/05/soap-envelope" 
+    xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd" 
+    xmlns:wsp="http://schemas.xmlsoap.org/ws/2004/09/policy" 
+    xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd" 
+    xmlns:wsa="http://www.w3.org/2005/08/addressing" 
+    xmlns:wst="http://schemas.xmlsoap.org/ws/2005/02/trust">
+    <S:Header>
+        <wsa:Action S:mustUnderstand="1">http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</wsa:Action>
+        <wsa:To S:mustUnderstand="1">https://login.microsoftonline.com/rst2.srf</wsa:To>
+        <ps:AuthInfo xmlns:ps="http://schemas.microsoft.com/LiveID/SoapServices/v1" Id="PPAuthInfo">
+            <ps:BinaryVersion>5</ps:BinaryVersion>
+            <ps:HostingApp>Managed IDCRL</ps:HostingApp>
+        </ps:AuthInfo>
+        <wsse:Security></wsse:Security>
+    </S:Header>
+    <S:Body>
+        <wst:RequestSecurityToken xmlns:wst="http://schemas.xmlsoap.org/ws/2005/02/trust" Id="RST0">
+            <wst:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</wst:RequestType>
+            <wsp:AppliesTo>
+                <wsa:EndpointReference>
+                    <wsa:Address>sharepoint.com</wsa:Address>
+                </wsa:EndpointReference>
+            </wsp:AppliesTo>
+            <wsp:PolicyReference URI="MBI"></wsp:PolicyReference>
+        </wst:RequestSecurityToken>
+    </S:Body>
+</S:Envelope>

office365/runtime/auth/SAML.xml

@@ -7,16 +7,16 @@
     <a:To s:mustUnderstand="1">https://login.microsoftonline.com/extSTS.srf</a:To>
     <o:Security s:mustUnderstand="1" xmlns:o="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd">
       <o:UsernameToken>
-        <o:Username>[username]</o:Username>
-        <o:Password>[password]</o:Password>
+        <o:Username>{username}</o:Username>
+        <o:Password>{password}</o:Password>
       </o:UsernameToken>
     </o:Security>
   </s:Header>
   <s:Body>
     <t:RequestSecurityToken xmlns:t="http://schemas.xmlsoap.org/ws/2005/02/trust">
       <wsp:AppliesTo xmlns:wsp="http://schemas.xmlsoap.org/ws/2004/09/policy">
         <a:EndpointReference>
-          <a:Address>[endpoint]</a:Address>
+          <a:Address>{auth_url}</a:Address>
         </a:EndpointReference>
       </wsp:AppliesTo>
       <wsp:PolicyReference xmlns:wsp="http://schemas.xmlsoap.org/ws/2004/09/policy" URI="MBI"></wsp:PolicyReference>

office365/runtime/auth/saml_token_provider.py

@@ -1,66 +1,78 @@
 import os
+import uuid
 from xml.etree import ElementTree
-
 import requests
 import requests.utils
-
 import office365.logger
 from office365.runtime.auth.base_token_provider import BaseTokenProvider
+from office365.runtime.auth.sts_info import STSInfo
+from office365.runtime.auth.user_realm_info import UserRealmInfo
 
 office365.logger.ensure_debug_secrets()
 
 
 class SamlTokenProvider(BaseTokenProvider, office365.logger.LoggerContext):
-    """ SAML Security Token Service for O365"""
-
-    def __init__(self, url, username, password):
-
-        self.url = url
-        self.username = username
-        self.password = password
-
-        # External Security Token Service for SPO
-        self.sts = {
-            'host': 'login.microsoftonline.com',
-            'path': '/extSTS.srf'
-        }
-
-        # Sign in page url
-        self.login = '/_forms/default.aspx?wa=wsignin1.0'
+    """SAML Security Token Service provider"""
 
+    def __init__(self, authority_url, username, password):
+        self.__username = username
+        self.__password = password
+        # Security Token Service info
+        self.sts = STSInfo(authority_url)
         # Last occurred error
         self.error = ''
-
-        self.token = None
         self.FedAuth = None
         self.rtFa = None
+        self._auth_cookies = []
+        self.__ns_prefixes = {
+            'S': '{http://www.w3.org/2003/05/soap-envelope}',
+            's': '{http://www.w3.org/2003/05/soap-envelope}',
+            'psf': '{http://schemas.microsoft.com/Passport/SoapServices/SOAPFault}',
+            'wst': '{http://schemas.xmlsoap.org/ws/2005/02/trust}',
+            'wsse': '{http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd}',
+            'saml': '{urn:oasis:names:tc:SAML:1.0:assertion}',
+            'u': '{http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd}',
+            'wsa': '{http://www.w3.org/2005/08/addressing}',
+            'wsp': '{http://schemas.xmlsoap.org/ws/2004/09/policy}',
+            'ps': '{http://schemas.microsoft.com/LiveID/SoapServices/v1}',
+            'ds': '{http://www.w3.org/2000/09/xmldsig#}'
+        }
+        for key in self.__ns_prefixes.keys():
+            ElementTree.register_namespace(key, self.__ns_prefixes[key][1:-1])
 
-    def acquire_token(self):
-        """Acquire user token"""
+    def acquire_token(self, **kwargs):
+        """Acquire user token
+        """
         logger = self.logger(self.acquire_token.__name__)
-        logger.debug('called')
+        logger.debug('acquire_token called')
 
         try:
             logger.debug("Acquiring Access Token..")
-            try:
-                from urlparse import urlparse  # Python 2.X
-            except ImportError:
-                from urllib.parse import urlparse  # Python 3+
-            url = urlparse(self.url)
-            options = {
-                'username': self.username,
-                'password': self.password,
-                'sts': self.sts,
-                'endpoint': url.scheme + '://' + url.hostname + self.login
-            }
-
-            self.acquire_service_token(options)
-            self.acquire_authentication_cookie(options)
+            user_realm = self.get_user_realm(self.__username)
+            if user_realm.IsFederated:
+                token = self.acquire_service_token_from_adfs(user_realm.STSAuthUrl, self.__username, self.__password)
+            else:
+                token = self.acquire_service_token(self.__username, self.__password)
+            self.acquire_authentication_cookie(token)
             return True
         except requests.exceptions.RequestException as e:
             self.error = "Error: {}".format(e)
             return False
 
+    def get_user_realm(self, login):
+        """Get User Realm"""
+        response = requests.post(self.sts.userRealmServiceUrl, data="login={0}&xml=1".format(login),
+                                 headers={'Content-Type': 'application/x-www-form-urlencoded'})
+        xml = ElementTree.fromstring(response.content)
+        node = xml.find('NameSpaceType')
+        if node is not None:
+            if node.text == 'Federated':
+                info = UserRealmInfo(xml.find('STSAuthURL').text, True)
+            else:
+                info = UserRealmInfo(None, False)
+            return info
+        return None
+
     def get_authentication_cookie(self):
         """Generate Auth Cookie"""
         logger = self.logger(self.get_authentication_cookie.__name__)
@@ -71,62 +83,112 @@ def get_authentication_cookie(self):
     def get_last_error(self):
         return self.error
 
-    def acquire_service_token(self, options):
+    def acquire_service_token_from_adfs(self, adfs_url, username, password):
+        logger = self.logger(self.acquire_service_token_from_adfs.__name__)
+        payload = self._prepare_request_from_template('FederatedSAML.xml', {
+            'auth_url': adfs_url,
+            'username': username,
+            'password': password,
+            'message_id': str(uuid.uuid4()),
+            'created': self.sts.created,
+            'expires': self.sts.expires,
+            'issuer': self.sts.federationTokenIssuer
+        })
+        response = requests.post(adfs_url, data=payload,
+                                 headers={'Content-Type': 'application/soap+xml; charset=utf-8'})
+        try:
+            xml = ElementTree.fromstring(response.content)
+            # 1.find assertion
+            assertion_node = xml.find(
+                '{0}Body/{1}RequestSecurityTokenResponse/{1}RequestedSecurityToken/{2}Assertion'.format(
+                    self.__ns_prefixes['s'], self.__ns_prefixes['wst'], self.__ns_prefixes['saml']))
+            if assertion_node is None:
+                self.error = 'Cannot get security assertion for user {0} from {1}'.format(self.__username, adfs_url)
+                logger.error(self.error)
+                return None
+            # 2. prepare & submit token request
+            self.sts.securityTokenServicePath = 'rst2.srf'
+            template = self._prepare_request_from_template('RST2.xml', {
+                'auth_url': self.sts.authorityUrl,
+                'serviceTokenUrl': self.sts.securityTokenServiceUrl
+            })
+            template_xml = ElementTree.fromstring(template)
+            security_node = template_xml.find(
+                '{0}Header/{1}Security'.format(self.__ns_prefixes['s'], self.__ns_prefixes['wsse']))
+
+            security_node.insert(1, assertion_node)
+            payload = ElementTree.tostring(template_xml).decode()
+            # 3. get token
+            response = requests.post(self.sts.securityTokenServiceUrl, data=payload,
+                                     headers={'Content-Type': 'application/soap+xml'})
+            token = self._process_service_token_response(response)
+            logger.debug_secrets('security token: %s', token)
+            return token
+        except ElementTree.ParseError as e:
+            self.error = 'An error occurred while parsing the server response: {}'.format(e)
+            logger.error(self.error)
+            return None
+
+    def acquire_service_token(self, username, password, service_target=None, service_policy=None):
         """Retrieve service token"""
         logger = self.logger(self.acquire_service_token.__name__)
-        logger.debug_secrets('options: %s', options)
-
-        request_body = self.prepare_security_token_request({
-            'username': options['username'],
-            'password': options['password'],
-            'endpoint': self.url
+        payload = self._prepare_request_from_template('SAML.xml', {
+            'auth_url': self.sts.authorityUrl,
+            'username': username,
+            'password': password,
+            'message_id': str(uuid.uuid4()),
+            'created': self.sts.created,
+            'expires': self.sts.expires,
+            'issuer': self.sts.federationTokenIssuer
         })
-
-        sts_url = 'https://' + options['sts']['host'] + options['sts']['path']
-        response = requests.post(sts_url, data=request_body,
+        logger.debug_secrets('options: %s', payload)
+        response = requests.post(self.sts.securityTokenServiceUrl, data=payload,
                                  headers={'Content-Type': 'application/x-www-form-urlencoded'})
-        token = self.process_service_token_response(response)
-        logger.debug_secrets('token: %s', token)
-        if token:
-            self.token = token
-            return True
-        return False
+        token = self._process_service_token_response(response)
+        logger.debug_secrets('security token: %s', token)
+        return token
 
-    def process_service_token_response(self, response):
-        logger = self.logger(self.process_service_token_response.__name__)
+    def _process_service_token_response(self, response):
+        logger = self.logger(self._process_service_token_response.__name__)
         logger.debug_secrets('response: %s\nresponse.content: %s', response, response.content)
 
-        xml = ElementTree.fromstring(response.content)
-        ns_prefixes = {'S': '{http://www.w3.org/2003/05/soap-envelope}',
-                       'psf': '{http://schemas.microsoft.com/Passport/SoapServices/SOAPFault}',
-                       'wst': '{http://schemas.xmlsoap.org/ws/2005/02/trust}',
-                       'wsse': '{http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd}'}
-        logger.debug_secrets("ns_prefixes: %s", ns_prefixes)
+        try:
+            xml = ElementTree.fromstring(response.content)
+        except ElementTree.ParseError as e:
+            self.error = 'An error occurred while parsing the server response: {}'.format(e)
+            logger.error(self.error)
+            return None
 
         # check for errors
-        if xml.find('{0}Body/{0}Fault'.format(ns_prefixes['S'])) is not None:
-            error = xml.find('{0}Body/{0}Fault/{0}Detail/{1}error/{1}internalerror/{1}text'.format(ns_prefixes['S'],
-                                                                                                   ns_prefixes['psf']))
-            self.error = 'An error occurred while retrieving token: {0}'.format(error.text)
+        if xml.find('{0}Body/{0}Fault'.format(self.__ns_prefixes['s'])) is not None:
+            error = xml.find('{0}Body/{0}Fault/{0}Detail/{1}error/{1}internalerror/{1}text'.format(self.__ns_prefixes['s'],
+                                                                                                   self.__ns_prefixes['psf']))
+            if error is None:
+                self.error = 'An error occurred while retrieving token from XML response.'
+            else:
+                self.error = 'An error occurred while retrieving token from XML response: {0}'.format(error.text)
             logger.error(self.error)
             return None
 
         # extract token
         token = xml.find(
             '{0}Body/{1}RequestSecurityTokenResponse/{1}RequestedSecurityToken/{2}BinarySecurityToken'.format(
-                ns_prefixes['S'], ns_prefixes['wst'], ns_prefixes['wsse']))
+                self.__ns_prefixes['s'], self.__ns_prefixes['wst'], self.__ns_prefixes['wsse']))
+        if token is None:
+            self.error = 'Cannot get binary security token for from {0}'.format(self.sts.securityTokenServiceUrl)
+            logger.error(self.error)
+            return None
         logger.debug_secrets("token: %s", token)
         return token.text
 
-    def acquire_authentication_cookie(self, options):
+    def acquire_authentication_cookie(self, security_token):
         """Retrieve SPO auth cookie"""
         logger = self.logger(self.acquire_authentication_cookie.__name__)
-
-        url = options['endpoint']
-
         session = requests.session()
-        logger.debug_secrets("session: %s\nsession.post(%s, data=%s)", session, url, self.token)
-        session.post(url, data=self.token, headers={'Content-Type': 'application/x-www-form-urlencoded'})
+        logger.debug_secrets("session: %s\nsession.post(%s, data=%s)", session, self.sts.signInPageUrl,
+                             security_token)
+        session.post(self.sts.signInPageUrl, data=security_token,
+                     headers={'Content-Type': 'application/x-www-form-urlencoded'})
         logger.debug_secrets("session.cookies: %s", session.cookies)
         cookies = requests.utils.dict_from_cookiejar(session.cookies)
         logger.debug_secrets("cookies: %s", cookies)
@@ -139,16 +201,15 @@ def acquire_authentication_cookie(self, options):
         return False
 
     @staticmethod
-    def prepare_security_token_request(params):
+    def _prepare_request_from_template(template_name, params):
         """Construct the request body to acquire security token from STS endpoint"""
-        logger = SamlTokenProvider.logger(SamlTokenProvider.prepare_security_token_request.__name__)
+        logger = SamlTokenProvider.logger()
         logger.debug_secrets('params: %s', params)
-
-        f = open(os.path.join(os.path.dirname(__file__), 'SAML.xml'))
+        f = open(os.path.join(os.path.dirname(__file__), template_name))
         try:
             data = f.read()
             for key in params:
-                data = data.replace('[' + key + ']', params[key])
+                data = data.replace('{' + key + '}', str(params[key]))
             return data
         finally:
             f.close()

office365/runtime/auth/sts_info.py

@@ -0,0 +1,25 @@
+import datetime
+
+
+class STSInfo(object):
+
+    def __init__(self, authority_url):
+        self.authorityUrl = authority_url
+        self.serviceUrl = 'https://login.microsoftonline.com'
+        self.securityTokenServicePath = 'extSTS.srf'
+        self.userRealmServicePath = 'GetUserRealm.srf'
+        self.federationTokenIssuer = 'urn:federation:MicrosoftOnline'
+        self.created = datetime.datetime.now()
+        self.expires = self.created + datetime.timedelta(minutes=10)
+
+    @property
+    def securityTokenServiceUrl(self):
+        return "/".join([self.serviceUrl, self.securityTokenServicePath])
+
+    @property
+    def signInPageUrl(self):
+        return "/".join([self.authorityUrl, '_forms/default.aspx?wa=wsignin1.0'])
+
+    @property
+    def userRealmServiceUrl(self):
+        return '/'.join([self.serviceUrl, self.userRealmServicePath])