feat: added basic dev mode support scripts

... including some fixes / patches to make the code work (fixed outdated lib
and disabled a part of the code that was crashing)

.gitignore

@@ -108,3 +108,4 @@ tests/resources/keys/*.pem
 .DS_Store
 .vscode
 .idea
+VENV_PATH

README.md

@@ -219,15 +219,6 @@ See [Fence and Google](docs/google_architecture.md) for more details on data acc
 
 ## Setup
 
-#### Install Requirements and Fence
-
-Install [Poetry](https://python-poetry.org/docs/#installation).
-
-```bash
-# Install Fence and dependencies
-poetry install
-```
-
 #### Create Configuration File
 
 Fence requires a configuration file to run. We have a command line
@@ -242,7 +233,7 @@ To create a new configuration file from the default configuration:
 python cfg_help.py create
 ```
 
-This file will be placed in one of the default search directories for Fence.
+This file will be placed in one of the default search directories for Fence (e.g. `~/.gen3/fence/fence-config.yaml`).
 
 To get the exact path where the new configuration file was created, use:
 
@@ -253,6 +244,54 @@ python cfg_help.py get
 The file should have detailed information about each of the configuration
 variables. **Remember to fill out the new configuration file!**
 
+#### Using Docker
+
+Build image using:
+
+```bash
+docker build . -t fence
+```
+
+:information_source: before running the command below,
+make sure you have the desired configuration in the .yaml and
+the necessary .pem files (see section **"Keypair Configuration"**) .
+
+Run with:
+
+```bash
+docker run --rm \
+-v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml:ro \
+-v /path/to/keys-folder-with-pem-files/:/fence/keys/folder/:ro \
+--name=fence \
+-p 80:80 \
+fence
+```
+
+Setup `fence_test` DB with:
+
+```bash
+ ./init_dev_db.sh
+```
+
+#### Using Poetry
+
+Install [Poetry](https://python-poetry.org/docs/#installation).
+
+Build with:
+
+```bash
+# Enter virtual environment
+poetry shell
+# Install Fence and dependencies
+poetry install
+```
+
+Run with:
+
+```bash
+And `run.py`
+```
+
 ##### Other Configuration Notes
 
 * Fence will look for configuration files from a list of search directories (

docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3.9'
+
+
+services:
+  # MEANT FOR LOCAL DEV ONLY
+  fence:
+    image: fence
+    container_name: fence
+    ports:
+      - "80:80"
+    volumes:
+      - $PWD/fence-config.yaml:/var/www/fence/fence-config.yaml:ro
+      - $PWD/keys/:/fence/keys:ro
+
+  fence_db:
+    image: postgres:12.10-bullseye
+    container_name: fence_db
+    environment:
+      POSTGRES_PASSWORD: mysecretpassword  # pragma: allowlist secret

fence-config.yaml

@@ -0,0 +1,273 @@
+---
+############################### Fence Configuration ####################################
+# This file contains various configurations for the Fence microservice.
+#
+# README:
+# - This is initially configured for minimal local development with reasonable defaults.
+# - Descriptions for each of the configurations (if any) will be *above* the variable as
+#   comments.
+# - Some configuration variables will have examples commented out below them.
+# - This is broken up into 2 main sections for REQUIRED and OPTIONAL configurations.
+#     - Optional configs will note what features or endpoints they support
+# - Underneath each main section the variables are logically grouped under named
+#   sections.
+#
+# NOTE: Login is NOT ready out of the box. Fill out REQUIRED configurations first
+
+########################################################################################
+#                               REQUIRED CONFIGURATIONS                                #
+########################################################################################
+APP_NAME: 'Gen3 Data Commons'
+# Where fence microservice is deployed
+BASE_URL: 'http://fence'
+# postgres db to connect to
+# connection url format:
+#     postgresql://[user[:password]@][netloc][:port][/dbname]
+# can also be set via env var DB
+DB: 'postgresql://postgres:mysecretpassword@fence_db:5432/fence_test' # pragma: allowlist secret
+
+ENCRYPTION_KEY: ''
+
+DEBUG: true
+MOCK_AUTH: true
+MOCK_GOOGLE_AUTH: true
+DEV_LOGIN_COOKIE_NAME: "dev_login"
+
+MOCK_STORAGE: true
+AUTHLIB_INSECURE_TRANSPORT: true
+ENABLE_PROMETHEUS_METRICS: false
+SESSION_COOKIE_SECURE: true
+
+ENABLE_CSRF_PROTECTION: true
+WTF_CSRF_SECRET_KEY: '{{ENCRYPTION_KEY}}'
+
+ENABLE_DB_MIGRATION: true
+DB_MIGRATION_POSTGRES_LOCK_KEY: 100
+
+
+# //////////////////////////////////////////////////////////////////////////////////////
+# OPEN ID CONNECT (OIDC)
+#   - Fully configure at least one client so login works
+#   - WARNING: Be careful changing the *_ALLOWED_SCOPES as you can break basic
+#              and optional functionality
+# //////////////////////////////////////////////////////////////////////////////////////
+OPENID_CONNECT:
+  fence:
+    # this api_base_url should be the root url for the OTHER fence
+    # something like: https://example.com
+    api_base_url: ''
+    # this client_id and client_secret should be obtained by registering THIS fence as
+    # a new client of the OTHER fence
+    client_id: ''
+    client_secret: ''
+    client_kwargs:
+      # openid is required to use OIDC flow
+      scope: 'openid'
+      # callback after logging in through the other fence
+      redirect_uri: '{{BASE_URL}}/login/fence/login'
+    # The next 3 should not need to be changed if the provider is following
+    # Oauth2 endpoint naming conventions
+    authorize_url: '{{api_base_url}}/oauth2/authorize'
+    access_token_url: '{{api_base_url}}/oauth2/token'
+    refresh_token_url: '{{api_base_url}}/oauth2/token'
+    name: ''
+    # if mock is true, will fake a successful login response for login
+    # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only)
+    mock: true
+    mock_default_user: '[email protected]'
+    # this is needed to enable InCommon login, if some LOGIN_OPTIONS are configured with idp=fence and a list of shib_idps:
+    shibboleth_discovery_url: ''
+
+# these are the *possible* scopes a client can be given, NOT scopes that are
+# given to all clients. You can be more restrictive during client creation
+CLIENT_ALLOWED_SCOPES:
+  - "openid"
+  - "user"
+  - "data"
+  - "google_credentials"
+  - "google_service_account"
+  - "google_link"
+  - "ga4gh_passport_v1"
+
+# these are the scopes that CAN be included in a user's own access_token
+USER_ALLOWED_SCOPES:
+  - "fence"
+  - "openid"
+  - "user"
+  - "data"
+  - "admin"
+  - "google_credentials"
+  - "google_service_account"
+  - "google_link"
+  - "ga4gh_passport_v1"
+SESSION_ALLOWED_SCOPES:
+  - "openid"
+  - "user"
+  - "credentials"
+  - "data"
+  - "admin"
+  - "google_credentials"
+  - "google_service_account"
+  - "google_link"
+  - "ga4gh_passport_v1"
+
+LOGIN_OPTIONS: []
+
+DEFAULT_LOGIN_IDP: null
+
+DEFAULT_LOGIN_URL: '{{BASE_URL}}/login/google'
+LOGIN_REDIRECT_WHITELIST: []
+ENABLED_IDENTITY_PROVIDERS: {}
+
+OAUTH2_JWT_ALG: 'RS256'
+OAUTH2_JWT_ENABLED: true
+OAUTH2_JWT_ISS: '{{BASE_URL}}'
+OAUTH2_PROVIDER_ERROR_URI: '/api/oauth2/errors'
+APPLICATION_ROOT: '/user'
+
+ACCESS_TOKEN_COOKIE_NAME: "access_token"
+SESSION_COOKIE_NAME: "fence"
+
+SESSION_COOKIE_DOMAIN:
+
+OAUTH2_TOKEN_EXPIRES_IN:
+  "authorization_code": 1200
+  "implicit": 1200
+ACCESS_TOKEN_EXPIRES_IN: 1200
+REFRESH_TOKEN_EXPIRES_IN: 2592000
+SESSION_TIMEOUT: 900
+SESSION_LIFETIME: 28800
+GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN: 2592000
+GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN: 604800
+
+GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN: 86400
+MAX_PRESIGNED_URL_TTL: 3600
+MAX_API_KEY_TTL: 2592000
+MAX_ACCESS_TOKEN_TTL: 3600
+TOKEN_PROJECTS_CUTOFF: 10
+RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION: false
+GEN3_PASSPORT_EXPIRES_IN: 43200
+
+GA4GH_DRS_POSTED_PASSPORT_FIELD: "passports"
+PRIVACY_POLICY_URL: null
+OVERRIDE_NGINX_RATE_LIMIT: 18
+
+DEFAULT_BACKOFF_SETTINGS_MAX_TRIES: 3
+SUPPORT_EMAIL_FOR_ERRORS: null
+
+SHIBBOLETH_HEADER: ''
+SSO_URL: ''
+ITRUST_GLOBAL_LOGOUT: ''
+GOOGLE_BULK_UPDATES: false
+
+STORAGE_CREDENTIALS: {}
+
+AWS_CREDENTIALS: {}
+S3_BUCKETS: {}
+
+GS_BUCKETS: {}
+ALLOWED_DATA_UPLOAD_BUCKETS: []
+
+DATA_UPLOAD_BUCKET: ''
+
+HTTP_PROXY:
+  host: null
+  port: 3128
+INDEXD: null
+INDEXD_USERNAME: 'fence'
+INDEXD_PASSWORD: ''
+AZ_BLOB_CREDENTIALS:
+AZ_BLOB_CONTAINER_URL: 'https://myfakeblob.blob.core.net/my-fake-container/'
+ARBORIST: null
+AUDIT_SERVICE: 'http://audit-service'
+ENABLE_AUDIT_LOGS:
+  presigned_url: false
+  login: false
+
+PUSH_AUDIT_LOGS_CONFIG:
+  type: aws_sqs
+  aws_sqs_config:
+    sqs_url:
+    region:
+    aws_cred:
+
+CIRRUS_CFG:
+  GOOGLE_API_KEY: ''
+  GOOGLE_PROJECT_ID: ''
+  GOOGLE_APPLICATION_CREDENTIALS: ''
+  GOOGLE_STORAGE_CREDS: ''
+  GOOGLE_ADMIN_EMAIL: ''
+  GOOGLE_IDENTITY_DOMAIN: ''
+  GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL: ''
+GOOGLE_GROUP_PREFIX: ''
+
+GOOGLE_SERVICE_ACCOUNT_PREFIX: ''
+BILLING_PROJECT_FOR_SIGNED_URLS:
+BILLING_PROJECT_FOR_SA_CREDS:
+ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS: false
+ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS: false
+
+GUN_MAIL:
+  'datacommons':
+    smtp_hostname: 'smtp.some.org'
+    api_key: ''
+    default_login: '[email protected]'
+    api_url: 'https://api.some.net/v3/some.example.com'
+    smtp_password: ''
+EMAIL_SERVER: 'localhost'
+SEND_FROM: '[email protected]'
+SEND_TO: '[email protected]'
+ALLOW_GOOGLE_LINKING: true
+WHITE_LISTED_GOOGLE_PARENT_ORGS: []
+
+WHITE_LISTED_SERVICE_ACCOUNT_EMAILS: []
+
+REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION:
+  enable: false
+  # this domain MUST exist in GUN_MAIL config
+  domain: 'example.com'
+  from: '[email protected]'
+  subject: 'User service account removal notification'
+  # the {} gets replaced dynamically in the Python code to be the Project ID
+  content: >
+    Some content.
+  #
+  # WARNING: This is NOT a bcc so the email is visible to the end-user
+  admin:
+    - '[email protected]'
+
+PROBLEM_USER_EMAIL_NOTIFICATION:
+  # this domain MUST exist in GUN_MAIL config
+  domain: 'example.com'
+  from: '[email protected]'
+  subject: 'Account access error notification'
+  # the {} gets replaced dynamically in the Python code to be the Project ID
+  content: >
+    Some content.
+  #
+  # WARNING: This is NOT a bcc so the email is visible to the end-user
+  admin:
+    - '[email protected]'
+GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS: []
+
+ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS: []
+
+MAX_ROLE_SESSION_INCREASE: false
+ASSUME_ROLE_CACHE_SECONDS: 1800
+REGISTER_USERS_ON: false
+REGISTERED_USERS_GROUP: ''
+SERVICE_ACCOUNT_LIMIT: 6
+
+GA4GH_PASSPORTS_TO_DRS_ENABLED: false
+RAS_REFRESH_EXPIRATION: 1296000
+
+EXPIRED_AUTHZ_REMOVAL_JOB_FREQ_IN_SECONDS: 300
+
+GLOBAL_PARSE_VISAS_ON_LOGIN: false
+
+ENABLE_VISA_UPDATE_CRON: false
+USERSYNC:
+  visa_types:
+    ras: []
+RAS_USERINFO_ENDPOINT: ''
+CLIENT_CREDENTIALS_ON_DOWNLOAD_ENABLED: false

fence/config.py

@@ -143,7 +143,8 @@ def post_process(self):
                 "Visa parsing on login is enabled but `ENABLE_VISA_UPDATE_CRON` is disabled!"
             )
 
-        self._validate_parent_child_studies(self._configs["dbGaP"])
+        # if self._configs["dbGaP"]:
+        #     self._validate_parent_child_studies(self._configs["dbGaP"])
 
     @staticmethod
     def _validate_parent_child_studies(dbgap_configs):