diff --git a/tests/ci_commands_script.sh b/bin/ci_commands_script.sh similarity index 68% rename from tests/ci_commands_script.sh rename to bin/ci_commands_script.sh index fef98a668..56e840448 100755 --- a/tests/ci_commands_script.sh +++ b/bin/ci_commands_script.sh @@ -2,4 +2,4 @@ mkdir -p /var/tmp/uwsgi_flask_metrics/ || true export PROMETHEUS_MULTIPROC_DIR="/var/tmp/uwsgi_flask_metrics/" -poetry run pytest -vv --cov=fence --cov-report xml tests +poetry run pytest -vv --cov=fence --cov-report xml ../tests diff --git a/fence/__init__.py b/fence/__init__.py index e1aec601d..1bab83020 100755 --- a/fence/__init__.py +++ b/fence/__init__.py @@ -18,14 +18,8 @@ logger = get_logger(__name__, log_level="debug") # Load the configuration *before* importing modules that rely on it -from fence.config import config +from fence.config import config, get_SQLAlchemyDriver from fence.settings import CONFIG_SEARCH_FOLDERS - -config.load( - config_path=os.environ.get("FENCE_CONFIG_PATH"), - search_folders=CONFIG_SEARCH_FOLDERS, -) - from fence.auth import logout, build_redirect_url from fence.metrics import metrics from fence.blueprints.data.indexd import S3IndexedFileLocation @@ -48,7 +42,6 @@ from fence.resources.storage import StorageManager from fence.resources.user.user_session import UserSessionInterface from fence.error_handler import get_error_response -from fence.utils import get_SQLAlchemyDriver import fence.blueprints.admin import fence.blueprints.data import fence.blueprints.login diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 85f2c021b..50e8bdb79 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -6,7 +6,6 @@ from urllib.parse import urlparse, ParseResult, urlunparse from datetime import datetime, timedelta -from sqlalchemy.sql.functions import user from cached_property import cached_property import gen3cirrus from gen3cirrus import GoogleCloudManager diff --git a/fence/config-default.yaml b/fence/config-default.yaml index 721994bde..f357b9b67 100755 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -545,6 +545,7 @@ dbGaP: # providing a list of additional_allowed_project_id_patterns allows usersyn to # read any filename that matches the patterns in the list. allow_non_dbGaP_whitelist: false + # Additional allowed patterns for project_ids. The default value in usersync is 'phs(\d{6}) for dbgap projects' additional_allowed_project_id_patterns: [] # parse out the consent from the dbgap accession number such that something # like "phs000123.v1.p1.c2" becomes "phs000123.c2". @@ -602,8 +603,6 @@ dbGaP: # 'studyZ': ['/orgD/'] # Allowed patterns for project_ids. The default value in usersync is 'phs(\d{6}) for dbgap projects' allowed_project_id_patterns: [] - # Additional allowed patterns for project_ids. The default value in usersync is 'phs(\d{6}) for dbgap projects' - additional_allowed_project_id_patterns: [] # Regex to match an assession number that has consent information in forms like: # phs00301123.c999 # phs000123.v3.p1.c3 diff --git a/fence/config.py b/fence/config.py index 7fa47c7cd..609580bf9 100644 --- a/fence/config.py +++ b/fence/config.py @@ -1,4 +1,10 @@ import os +from functools import wraps + +import bcrypt +import flask +import requests +from werkzeug.datastructures import ImmutableMultiDict from yaml import safe_load as yaml_load import urllib.parse @@ -6,6 +12,24 @@ from gen3config import Config from cdislogging import get_logger +from fence.settings import CONFIG_SEARCH_FOLDERS +from fence.utils import log_backoff_retry, log_backoff_giveup, exception_do_not_retry, generate_client_credentials, \ + logger +from fence.models import Client, User, query_for_user +from fence.errors import NotFound +from userdatamodel import Base +from sqlalchemy import ( + Integer, + String, + Column, + text, + event, + ForeignKey) +from sqlalchemy.orm import relationship, backref +from sqlalchemy.dialects.postgresql import JSONB +from userdatamodel.models import ( + IdentityProvider, + User) logger = get_logger(__name__) @@ -77,8 +101,8 @@ def post_process(self): # NOTE: use when fence will be deployed in such a way that fence will # only receive traffic from internal clients, and can safely use HTTP if ( - self._configs.get("AUTHLIB_INSECURE_TRANSPORT") - and "AUTHLIB_INSECURE_TRANSPORT" not in os.environ + self._configs.get("AUTHLIB_INSECURE_TRANSPORT") + and "AUTHLIB_INSECURE_TRANSPORT" not in os.environ ): os.environ["AUTHLIB_INSECURE_TRANSPORT"] = "true" @@ -100,11 +124,11 @@ def post_process(self): # billing rights (in other words, only use it when interacting with buckets # Fence is aware of) if self._configs.get("BILLING_PROJECT_FOR_SA_CREDS") or self._configs.get( - "BILLING_PROJECT_FOR_SIGNED_URLS" + "BILLING_PROJECT_FOR_SIGNED_URLS" ): if ( - "USER_ALLOWED_SCOPES" in self._configs - and "google_credentials" in self._configs["USER_ALLOWED_SCOPES"] + "USER_ALLOWED_SCOPES" in self._configs + and "google_credentials" in self._configs["USER_ALLOWED_SCOPES"] ): logger.warning( "Configuration does not restrict end-user access to billing. Correcting. " @@ -117,8 +141,8 @@ def post_process(self): self._configs["USER_ALLOWED_SCOPES"].remove("google_credentials") if ( - "SESSION_ALLOWED_SCOPES" in self._configs - and "google_credentials" in self._configs["SESSION_ALLOWED_SCOPES"] + "SESSION_ALLOWED_SCOPES" in self._configs + and "google_credentials" in self._configs["SESSION_ALLOWED_SCOPES"] ): logger.warning( "Configuration does not restrict end-user access to billing. Correcting. " @@ -131,8 +155,8 @@ def post_process(self): self._configs["SESSION_ALLOWED_SCOPES"].remove("google_credentials") if ( - not self._configs["ENABLE_VISA_UPDATE_CRON"] - and self._configs["GLOBAL_PARSE_VISAS_ON_LOGIN"] is not False + not self._configs["ENABLE_VISA_UPDATE_CRON"] + and self._configs["GLOBAL_PARSE_VISAS_ON_LOGIN"] is not False ): raise Exception( "Visa parsing on login is enabled but `ENABLE_VISA_UPDATE_CRON` is disabled!" @@ -169,3 +193,236 @@ def _validate_parent_child_studies(dbgap_configs): config = FenceConfig(DEFAULT_CFG_PATH) +config.load(config_path=os.environ.get("FENCE_CONFIG_PATH"), search_folders=CONFIG_SEARCH_FOLDERS,) +DEFAULT_BACKOFF_SETTINGS = { + "on_backoff": log_backoff_retry, + "on_giveup": log_backoff_giveup, + # todo: config does not have this key but it probably should. why broken? + "max_tries": config.get("DEFAULT_BACKOFF_SETTINGS_MAX_TRIES", 3), + "giveup": exception_do_not_retry, +} + + +def send_email(from_email, to_emails, subject, text, smtp_domain): + """ + Send email to group of emails using mail gun api. + + https://app.mailgun.com/ + + Args: + from_email(str): from email + to_emails(list): list of emails to receive the messages + text(str): the text message + smtp_domain(dict): smtp domain server + + { + "smtp_hostname": "smtp.mailgun.org", + "default_login": "postmaster@mailgun.planx-pla.net", + "api_url": "https://api.mailgun.net/v3/mailgun.planx-pla.net", + "smtp_password": "password", # pragma: allowlist secret + "api_key": "api key" # pragma: allowlist secret + } + + Returns: + Http response + + Exceptions: + KeyError + + """ + if smtp_domain not in config["GUN_MAIL"] or not config["GUN_MAIL"].get( + smtp_domain + ).get("smtp_password"): + raise NotFound( + "SMTP Domain '{}' does not exist in configuration for GUN_MAIL or " + "smtp_password was not provided. " + "Cannot send email.".format(smtp_domain) + ) + + api_key = config["GUN_MAIL"][smtp_domain].get("api_key", "") + email_url = config["GUN_MAIL"][smtp_domain].get("api_url", "") + "/messages" + + return requests.post( + email_url, + auth=("api", api_key), + data={"from": from_email, "to": to_emails, "subject": subject, "text": text}, + ) + + +def create_client( + DB, + username=None, + urls=[], + name="", + description="", + auto_approve=False, + is_admin=False, + grant_types=None, + confidential=True, + arborist=None, + policies=None, + allowed_scopes=None, + expires_in=None, +): + client_id, client_secret, hashed_secret = generate_client_credentials(confidential) + if arborist is not None: + arborist.create_client(client_id, policies) + driver = get_SQLAlchemyDriver(DB) + auth_method = "client_secret_basic" if confidential else "none" + + allowed_scopes = allowed_scopes or config["CLIENT_ALLOWED_SCOPES"] + if not set(allowed_scopes).issubset(set(config["CLIENT_ALLOWED_SCOPES"])): + raise ValueError( + "Each allowed scope must be one of: {}".format( + config["CLIENT_ALLOWED_SCOPES"] + ) + ) + + if "openid" not in allowed_scopes: + allowed_scopes.append("openid") + logger.warning('Adding required "openid" scope to list of allowed scopes.') + + with driver.session as s: + user = None + if username: + user = query_for_user(session=s, username=username) + if not user: + user = User(username=username, is_admin=is_admin) + s.add(user) + + if s.query(Client).filter(Client.name == name).first(): + if arborist is not None: + arborist.delete_client(client_id) + raise Exception("client {} already exists".format(name)) + + client = Client( + client_id=client_id, + client_secret=hashed_secret, + user=user, + redirect_uris=urls, + allowed_scopes=" ".join(allowed_scopes), + description=description, + name=name, + auto_approve=auto_approve, + grant_types=grant_types, + is_confidential=confidential, + token_endpoint_auth_method=auth_method, + expires_in=expires_in, + ) + s.add(client) + s.commit() + + return client_id, client_secret + + +def hash_secret(f): + @wraps(f) + def wrapper(*args, **kwargs): + has_secret = "client_secret" in flask.request.form + has_client_id = "client_id" in flask.request.form + if flask.request.form and has_secret and has_client_id: + form = flask.request.form.to_dict() + with flask.current_app.db.session as session: + client = ( + session.query(Client) + .filter(Client.client_id == form["client_id"]) + .first() + ) + if client: + form["client_secret"] = bcrypt.hashpw( + form["client_secret"].encode("utf-8"), + client.client_secret.encode("utf-8"), + ).decode("utf-8") + flask.request.form = ImmutableMultiDict(form) + + return f(*args, **kwargs) + + return wrapper + + +def get_SQLAlchemyDriver(db_conn_url): + from userdatamodel.driver import SQLAlchemyDriver + + # override userdatamodel's `setup_db` function which creates tables + # and runs database migrations, because Alembic handles that now. + # TODO move userdatamodel code to Fence and remove dependencies to it + SQLAlchemyDriver.setup_db = lambda _: None + return SQLAlchemyDriver(db_conn_url) + + +def get_issuer_to_idp(): + possibly_matching_idps = [IdentityProvider.ras] + issuer_to_idp = {} + + oidc = config.get("OPENID_CONNECT", {}) + for idp in possibly_matching_idps: + discovery_url = oidc.get(idp, {}).get("discovery_url") + if discovery_url: + for allowed_issuer in config["GA4GH_VISA_ISSUER_ALLOWLIST"]: + if discovery_url.startswith(allowed_issuer): + issuer_to_idp[allowed_issuer] = idp + break + + return issuer_to_idp + + +class IssSubPairToUser(Base): + # issuer & sub pair mapping to Gen3 User sub + + __tablename__ = "iss_sub_pair_to_user" + + iss = Column(String(), primary_key=True) + sub = Column(String(), primary_key=True) + + fk_to_User = Column( + Integer, ForeignKey(User.id, ondelete="CASCADE"), nullable=False + ) # foreign key for User table + user = relationship( + "User", + backref=backref( + "iss_sub_pairs", + cascade="all, delete-orphan", + passive_deletes=True, + ), + ) + + # dump whatever idp provides in here + extra_info = Column(JSONB(), server_default=text("'{}'")) + + +@event.listens_for(IssSubPairToUser.__table__, "after_create") +def populate_iss_sub_pair_to_user_table(target, connection, **kw): + """ + Populate iss_sub_pair_to_user table using User table's id_from_idp + column. + """ + issuer_to_idp = get_issuer_to_idp() + for issuer, idp_name in IssSubPairToUser.ISSUER_TO_IDP.items(): + logger.info( + 'Attempting to populate iss_sub_pair_to_user table for users with "{}" idp and "{}" issuer'.format( + idp_name, issuer + ) + ) + transaction = connection.begin() + try: + connection.execute( + text( + """ + WITH identity_provider_id AS (SELECT id FROM identity_provider WHERE name=:idp_name) + INSERT INTO iss_sub_pair_to_user (iss, sub, "fk_to_User", extra_info) + SELECT :iss, id_from_idp, id, additional_info + FROM "User" + WHERE idp_id IN (SELECT * FROM identity_provider_id) AND id_from_idp IS NOT NULL; + """ + ), + idp_name=idp_name, + iss=issuer, + ) + except Exception as e: + transaction.rollback() + logger.warning( + "Could not populate iss_sub_pair_to_user table: {}".format(e) + ) + else: + transaction.commit() + logger.info("Population was successful") diff --git a/fence/jwt/validate.py b/fence/jwt/validate.py index fa47979b4..a9bae02de 100644 --- a/fence/jwt/validate.py +++ b/fence/jwt/validate.py @@ -153,7 +153,7 @@ def validate_jwt( attempt_refresh=attempt_refresh, **kwargs ) - except Error as e: + except Exception as e: raise JWTError("Invalid refresh token: {}".format(e)) elif unverified_claims.get("pur") == "api_key" and isinstance( e, JWTAudienceError @@ -170,7 +170,7 @@ def validate_jwt( attempt_refresh=attempt_refresh, **kwargs ) - except Error as e: + except Exception as e: raise JWTError("Invalid API key: {}".format(e)) else: ##### end refresh token, API key patch block ##### diff --git a/fence/models.py b/fence/models.py index 779d27519..2bbdf908d 100644 --- a/fence/models.py +++ b/fence/models.py @@ -25,40 +25,20 @@ String, Column, Boolean, - Text, - text, - event, -) + Text) from sqlalchemy.dialects.postgresql import ARRAY, JSONB from sqlalchemy.orm import relationship, backref from sqlalchemy import func from sqlalchemy.schema import ForeignKey from userdatamodel import Base from userdatamodel.models import ( - AccessPrivilege, - Application, - AuthorizationProvider, Bucket, - Certificate, - CloudProvider, - ComputeAccess, GoogleProxyGroup, - Group, - HMACKeyPair, - HMACKeyPairArchive, IdentityProvider, Project, - ProjectToBucket, - S3Credential, - StorageAccess, - Tag, - User, - UserToBucket, - UserToGroup, -) + User) from fence import logger -from fence.config import config from fence.errors import UserError @@ -744,84 +724,3 @@ class UpstreamRefreshToken(Base): ) refresh_token = Column(Text, nullable=False) expires = Column(BigInteger, nullable=False) - - -class IssSubPairToUser(Base): - # issuer & sub pair mapping to Gen3 User sub - - __tablename__ = "iss_sub_pair_to_user" - - iss = Column(String(), primary_key=True) - sub = Column(String(), primary_key=True) - - fk_to_User = Column( - Integer, ForeignKey(User.id, ondelete="CASCADE"), nullable=False - ) # foreign key for User table - user = relationship( - "User", - backref=backref( - "iss_sub_pairs", - cascade="all, delete-orphan", - passive_deletes=True, - ), - ) - - # dump whatever idp provides in here - extra_info = Column(JSONB(), server_default=text("'{}'")) - - def _get_issuer_to_idp(): - possibly_matching_idps = [IdentityProvider.ras] - issuer_to_idp = {} - - oidc = config.get("OPENID_CONNECT", {}) - for idp in possibly_matching_idps: - discovery_url = oidc.get(idp, {}).get("discovery_url") - if discovery_url: - for allowed_issuer in config["GA4GH_VISA_ISSUER_ALLOWLIST"]: - if discovery_url.startswith(allowed_issuer): - issuer_to_idp[allowed_issuer] = idp - break - - return issuer_to_idp - - ISSUER_TO_IDP = _get_issuer_to_idp() - - # no longer need function since results stored in var - del _get_issuer_to_idp - - -@event.listens_for(IssSubPairToUser.__table__, "after_create") -def populate_iss_sub_pair_to_user_table(target, connection, **kw): - """ - Populate iss_sub_pair_to_user table using User table's id_from_idp - column. - """ - for issuer, idp_name in IssSubPairToUser.ISSUER_TO_IDP.items(): - logger.info( - 'Attempting to populate iss_sub_pair_to_user table for users with "{}" idp and "{}" issuer'.format( - idp_name, issuer - ) - ) - transaction = connection.begin() - try: - connection.execute( - text( - """ - WITH identity_provider_id AS (SELECT id FROM identity_provider WHERE name=:idp_name) - INSERT INTO iss_sub_pair_to_user (iss, sub, "fk_to_User", extra_info) - SELECT :iss, id_from_idp, id, additional_info - FROM "User" - WHERE idp_id IN (SELECT * FROM identity_provider_id) AND id_from_idp IS NOT NULL; - """ - ), - idp_name=idp_name, - iss=issuer, - ) - except Exception as e: - transaction.rollback() - logger.warning( - "Could not populate iss_sub_pair_to_user table: {}".format(e) - ) - else: - transaction.commit() - logger.info("Population was successful") diff --git a/fence/resources/audit/client.py b/fence/resources/audit/client.py index 9b90fa0d9..7733de564 100644 --- a/fence/resources/audit/client.py +++ b/fence/resources/audit/client.py @@ -4,10 +4,9 @@ import requests import traceback -from fence.config import config +from fence.config import config, DEFAULT_BACKOFF_SETTINGS from fence.errors import InternalError from fence.resources.audit.utils import is_audit_enabled -from fence.utils import DEFAULT_BACKOFF_SETTINGS class AuditServiceClient: diff --git a/fence/resources/google/access_utils.py b/fence/resources/google/access_utils.py index 002d342b1..eacc38ff7 100644 --- a/fence/resources/google/access_utils.py +++ b/fence/resources/google/access_utils.py @@ -11,16 +11,16 @@ from gen3cirrus.google_cloud.errors import GoogleAPIError from gen3cirrus.google_cloud.iam import GooglePolicy from gen3cirrus import GoogleCloudManager +from userdatamodel.user import AccessPrivilege import fence from cdislogging import get_logger -from fence.config import config -from fence.errors import NotFound, NotSupported +from fence.config import config, DEFAULT_BACKOFF_SETTINGS +from fence.errors import NotFound, NotSupported, UserError from fence.models import ( User, Project, - AccessPrivilege, UserGoogleAccount, UserServiceAccount, ServiceAccountAccessPrivilege, @@ -31,7 +31,7 @@ get_monitoring_service_account_email, is_google_managed_service_account, ) -from fence.utils import get_valid_expiration_from_request, DEFAULT_BACKOFF_SETTINGS +from fence.utils import get_valid_expiration_from_request logger = get_logger(__name__) diff --git a/fence/resources/google/utils.py b/fence/resources/google/utils.py index 5c45790ba..e491322f9 100644 --- a/fence/resources/google/utils.py +++ b/fence/resources/google/utils.py @@ -19,7 +19,7 @@ from userdatamodel.user import GoogleProxyGroup, User, AccessPrivilege from fence.auth import current_token -from fence.config import config +from fence.config import config, DEFAULT_BACKOFF_SETTINGS from fence.errors import NotSupported, InternalError, UserError from fence.models import ( GoogleServiceAccount, @@ -34,12 +34,10 @@ ) from fence.resources.google import STORAGE_ACCESS_PROVIDER_NAME from fence.errors import NotSupported, NotFound -from fence.utils import get_SQLAlchemyDriver +from fence import get_SQLAlchemyDriver from cdislogging import get_logger -from fence.utils import DEFAULT_BACKOFF_SETTINGS - logger = get_logger(__name__) diff --git a/fence/resources/openid/ras_oauth2.py b/fence/resources/openid/ras_oauth2.py index b94897b87..f88f0a6ff 100644 --- a/fence/resources/openid/ras_oauth2.py +++ b/fence/resources/openid/ras_oauth2.py @@ -7,26 +7,14 @@ import fence.resources.ga4gh.passports import fence.scripting.fence_create import fence.resources.ga4gh.passports - from flask import current_app from jose import jwt as jose_jwt - -from authutils.errors import JWTError -from authutils.token.core import get_iss, get_kid from gen3authz.client.arborist.errors import ArboristError - - -from fence.config import config +from fence.config import config, DEFAULT_BACKOFF_SETTINGS, IssSubPairToUser from fence.models import ( - GA4GHVisaV1, IdentityProvider, - User, - IssSubPairToUser, query_for_user, - create_user, -) -from fence.jwt.validate import validate_jwt -from fence.utils import DEFAULT_BACKOFF_SETTINGS + create_user) from fence.errors import InternalError from .idp_oauth2 import Oauth2ClientBase diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index a4b15aff8..ef80a36a8 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -54,14 +54,13 @@ get_client_expires_at, ) from fence.scripting.google_monitor import email_users_without_access, validation_check -from fence.config import config +from fence.config import config, create_client from fence.sync.sync_users import UserSyncer from fence.utils import ( - create_client, get_valid_expiration, generate_client_credentials, - get_SQLAlchemyDriver, ) +from fence import get_SQLAlchemyDriver from sqlalchemy.orm.attributes import flag_modified from gen3authz.client.arborist.client import ArboristClient diff --git a/fence/scripting/google_monitor.py b/fence/scripting/google_monitor.py index ad232519d..041f0d062 100644 --- a/fence/scripting/google_monitor.py +++ b/fence/scripting/google_monitor.py @@ -13,6 +13,7 @@ from cdislogging import get_logger +import fence.config from fence.resources.google.validity import ( GoogleProjectValidity, GoogleServiceAccountValidity, @@ -514,7 +515,7 @@ def _send_emails_informing_service_account_removal( for reason in removal_reasons: content += "\n\t\t\t - {}".format(reason) - return utils.send_email(from_email, to_emails, subject, content, domain) + return fence.config.send_email(from_email, to_emails, subject, content, domain) def _get_users_without_access(db, auth_ids, user_emails, check_linking): @@ -609,7 +610,7 @@ def email_user_without_access(user_email, projects, google_project_id): text = config["PROBLEM_USER_EMAIL_NOTIFICATION"]["content"] content = text.format(google_project_id, ",".join(projects)) - return utils.send_email(from_email, to_emails, subject, content, domain) + return fence.config.send_email(from_email, to_emails, subject, content, domain) def email_users_without_access( diff --git a/fence/settings.py b/fence/settings.py index 0254cde04..809051003 100644 --- a/fence/settings.py +++ b/fence/settings.py @@ -26,7 +26,7 @@ # located elsewhere, use that location in ``imp.load_source`` instead of # ``/var/www/local_settings.py``, just below. def use_deprecated_settings(): - ENCRYPTION_KEY = HMAC_ENCRYPTION_KEY + ENCRYPTION_KEY = "" # HMAC_ENCRYPTION_KEY try: diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 5a11c4ac6..41bad6a7a 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -26,7 +26,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy import func -from fence.config import config +from fence.config import config, DEFAULT_BACKOFF_SETTINGS from fence.models import ( AccessPrivilege, AuthorizationProvider, @@ -44,7 +44,7 @@ from fence.resources.google.access_utils import GoogleUpdateException from fence.sync import utils from fence.sync.passport_sync.ras_sync import RASVisa -from fence.utils import get_SQLAlchemyDriver, DEFAULT_BACKOFF_SETTINGS +from fence import get_SQLAlchemyDriver def _format_policy_id(path, privilege): diff --git a/fence/utils.py b/fence/utils.py index 463fb6f75..9941ee7bf 100644 --- a/fence/utils.py +++ b/fence/utils.py @@ -6,18 +6,14 @@ from random import SystemRandom import re import string -import requests from urllib.parse import urlencode from urllib.parse import parse_qs, urlsplit, urlunsplit import sys from cdislogging import get_logger import flask -from werkzeug.datastructures import ImmutableMultiDict -from fence.models import Client, User, query_for_user -from fence.errors import NotFound, UserError -from fence.config import config +from fence.errors import UserError from authlib.oauth2.rfc6749.util import scope_to_list from authlib.oauth2.rfc6749.errors import InvalidScopeError @@ -57,97 +53,6 @@ def generate_client_credentials(confidential): return client_id, client_secret, hashed_secret -def create_client( - DB, - username=None, - urls=[], - name="", - description="", - auto_approve=False, - is_admin=False, - grant_types=None, - confidential=True, - arborist=None, - policies=None, - allowed_scopes=None, - expires_in=None, -): - client_id, client_secret, hashed_secret = generate_client_credentials(confidential) - if arborist is not None: - arborist.create_client(client_id, policies) - driver = get_SQLAlchemyDriver(DB) - auth_method = "client_secret_basic" if confidential else "none" - - allowed_scopes = allowed_scopes or config["CLIENT_ALLOWED_SCOPES"] - if not set(allowed_scopes).issubset(set(config["CLIENT_ALLOWED_SCOPES"])): - raise ValueError( - "Each allowed scope must be one of: {}".format( - config["CLIENT_ALLOWED_SCOPES"] - ) - ) - - if "openid" not in allowed_scopes: - allowed_scopes.append("openid") - logger.warning('Adding required "openid" scope to list of allowed scopes.') - - with driver.session as s: - user = None - if username: - user = query_for_user(session=s, username=username) - if not user: - user = User(username=username, is_admin=is_admin) - s.add(user) - - if s.query(Client).filter(Client.name == name).first(): - if arborist is not None: - arborist.delete_client(client_id) - raise Exception("client {} already exists".format(name)) - - client = Client( - client_id=client_id, - client_secret=hashed_secret, - user=user, - redirect_uris=urls, - allowed_scopes=" ".join(allowed_scopes), - description=description, - name=name, - auto_approve=auto_approve, - grant_types=grant_types, - is_confidential=confidential, - token_endpoint_auth_method=auth_method, - expires_in=expires_in, - ) - s.add(client) - s.commit() - - return client_id, client_secret - - -def hash_secret(f): - @wraps(f) - def wrapper(*args, **kwargs): - has_secret = "client_secret" in flask.request.form - has_client_id = "client_id" in flask.request.form - if flask.request.form and has_secret and has_client_id: - form = flask.request.form.to_dict() - with flask.current_app.db.session as session: - client = ( - session.query(Client) - .filter(Client.client_id == form["client_id"]) - .first() - ) - if client: - form["client_secret"] = bcrypt.hashpw( - form["client_secret"].encode("utf-8"), - client.client_secret.encode("utf-8"), - ).decode("utf-8") - flask.request.form = ImmutableMultiDict(form) - - return f(*args, **kwargs) - - return wrapper - - def wrap_list_required(f): @wraps(f) def wrapper(d, *args, **kwargs): @@ -254,52 +159,6 @@ def split_url_and_query_params(url): return url, query_params -def send_email(from_email, to_emails, subject, text, smtp_domain): - """ - Send email to group of emails using mail gun api. - - https://app.mailgun.com/ - - Args: - from_email(str): from email - to_emails(list): list of emails to receive the messages - text(str): the text message - smtp_domain(dict): smtp domain server - - { - "smtp_hostname": "smtp.mailgun.org", - "default_login": "postmaster@mailgun.planx-pla.net", - "api_url": "https://api.mailgun.net/v3/mailgun.planx-pla.net", - "smtp_password": "password", # pragma: allowlist secret - "api_key": "api key" # pragma: allowlist secret - } - - Returns: - Http response - - Exceptions: - KeyError - - """ - if smtp_domain not in config["GUN_MAIL"] or not config["GUN_MAIL"].get( - smtp_domain - ).get("smtp_password"): - raise NotFound( - "SMTP Domain '{}' does not exist in configuration for GUN_MAIL or " - "smtp_password was not provided. " - "Cannot send email.".format(smtp_domain) - ) - - api_key = config["GUN_MAIL"][smtp_domain].get("api_key", "") - email_url = config["GUN_MAIL"][smtp_domain].get("api_url", "") + "/messages" - - return requests.post( - email_url, - auth=("api", api_key), - data={"from": from_email, "to": to_emails, "subject": subject, "text": text}, - ) - - def get_valid_expiration_from_request( expiry_param="expires_in", max_limit=None, default=None ): @@ -388,52 +247,7 @@ def _is_status(code): return False -def get_from_cache(item_id, memory_cache, db_cache_table, db_cache_table_id_field="id"): - """ - Attempt to get a cached item and store in memory cache from db if necessary. - - NOTE: This requires custom implementation for putting items in the db cache table. - """ - # try to retrieve from local in-memory cache - rv, expires_at = memory_cache.get(item_id, (None, 0)) - if expires_at > expiry: - return rv - - # try to retrieve from database cache - if hasattr(flask.current_app, "db"): # we don't have db in startup - with flask.current_app.db.session as session: - cache = ( - session.query(db_cache_table) - .filter( - getattr(db_cache_table, db_cache_table_id_field, None) == item_id - ) - .first() - ) - if cache and cache.expires_at and cache.expires_at > expiry: - rv = dict(cache) - - # store in memory cache - memory_cache[item_id] = rv, cache.expires_at - return rv - - -def get_SQLAlchemyDriver(db_conn_url): - from userdatamodel.driver import SQLAlchemyDriver - - # override userdatamodel's `setup_db` function which creates tables - # and runs database migrations, because Alembic handles that now. - # TODO move userdatamodel code to Fence and remove dependencies to it - SQLAlchemyDriver.setup_db = lambda _: None - return SQLAlchemyDriver(db_conn_url) - - # Default settings to control usage of backoff library. -DEFAULT_BACKOFF_SETTINGS = { - "on_backoff": log_backoff_retry, - "on_giveup": log_backoff_giveup, - "max_tries": config["DEFAULT_BACKOFF_SETTINGS_MAX_TRIES"], - "giveup": exception_do_not_retry, -} def validate_scopes(request_scopes, client): diff --git a/migrations/versions/e4c7b0ab68d3_create_tables.py b/migrations/versions/e4c7b0ab68d3_create_tables.py index 31d9346a9..81fa00997 100644 --- a/migrations/versions/e4c7b0ab68d3_create_tables.py +++ b/migrations/versions/e4c7b0ab68d3_create_tables.py @@ -15,7 +15,7 @@ from sqlalchemy.dialects import postgresql from bin.old_migration_script import migrate -from fence.utils import get_SQLAlchemyDriver +from fence import get_SQLAlchemyDriver # revision identifiers, used by Alembic. revision = "e4c7b0ab68d3" diff --git a/tests/conftest.py b/tests/conftest.py index 9baba01a1..94965864c 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,20 +19,14 @@ from addict import Dict from alembic.config import main as alembic_main from authutils.testing.fixtures import ( - _hazmat_rsa_private_key, - _hazmat_rsa_private_key_2, rsa_private_key, - rsa_private_key_2, - rsa_public_key, - rsa_public_key_2, -) + rsa_public_key) from cryptography.fernet import Fernet import bcrypt import jwt from mock import patch, MagicMock, PropertyMock import pytest import requests -from sqlalchemy.ext.compiler import compiles # Set FENCE_CONFIG_PATH *before* loading the configuration diff --git a/tests/dbgap_sync/test_user_sync.py b/tests/dbgap_sync/test_user_sync.py index f85cc28e5..a11eab0fb 100644 --- a/tests/dbgap_sync/test_user_sync.py +++ b/tests/dbgap_sync/test_user_sync.py @@ -9,9 +9,8 @@ from fence import models from fence.resources.google.access_utils import GoogleUpdateException -from fence.config import config +from fence.config import config, DEFAULT_BACKOFF_SETTINGS from fence.job.visa_update_cronjob import Visa_Token_Update -from fence.utils import DEFAULT_BACKOFF_SETTINGS from tests.dbgap_sync.conftest import ( get_test_encoded_decoded_visa_and_exp, diff --git a/tests/ga4gh/test_ga4gh.py b/tests/ga4gh/test_ga4gh.py index 0d22b7477..7376bed59 100644 --- a/tests/ga4gh/test_ga4gh.py +++ b/tests/ga4gh/test_ga4gh.py @@ -3,8 +3,8 @@ from cdislogging import get_logger -from fence.config import config -from fence.models import IdentityProvider, IssSubPairToUser +from fence.config import config, IssSubPairToUser +from fence.models import IdentityProvider from fence.resources.openid.ras_oauth2 import RASOauth2Client from fence.resources.ga4gh.passports import get_or_create_gen3_user_from_iss_sub diff --git a/tests/google/test_access_utils.py b/tests/google/test_access_utils.py index 8df13ec55..396d24956 100644 --- a/tests/google/test_access_utils.py +++ b/tests/google/test_access_utils.py @@ -26,7 +26,7 @@ patch_user_service_account, remove_white_listed_service_account_ids, update_google_groups_for_users, GoogleUpdateException ) -from fence.utils import DEFAULT_BACKOFF_SETTINGS +from fence.config import DEFAULT_BACKOFF_SETTINGS class MockResponse: diff --git a/tests/google/test_utils.py b/tests/google/test_utils.py index 61f0568b3..c5c9028db 100644 --- a/tests/google/test_utils.py +++ b/tests/google/test_utils.py @@ -6,7 +6,7 @@ give_service_account_billing_access_if_necessary, GoogleCloudManager, ) -from fence.utils import DEFAULT_BACKOFF_SETTINGS +from fence.config import DEFAULT_BACKOFF_SETTINGS def test_give_service_account_billing_access_if_necessary_fails(cloud_manager): diff --git a/tests/scripting/test_fence-create.py b/tests/scripting/test_fence-create.py index deae90d34..8ec6148df 100644 --- a/tests/scripting/test_fence-create.py +++ b/tests/scripting/test_fence-create.py @@ -12,7 +12,7 @@ from fence.config import config from fence.errors import UserError from fence.jwt.validate import validate_jwt -from fence.utils import create_client, get_SQLAlchemyDriver +from fence import get_SQLAlchemyDriver from fence.models import ( AccessPrivilege, Project,