diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 24226de..8b2eafd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,11 +1,11 @@ --- -name: Lint +name: Super-Linter on: push: - branches: [ master, devel ] + branches: [2.0-alpha-tests] pull_request: - branches: [ master ] + branches: [2.0-alpha-tests] workflow_dispatch: jobs: diff --git a/.github/workflows/publish_image.yml b/.github/workflows/publish_image.yml index f33ce98..fc7e278 100644 --- a/.github/workflows/publish_image.yml +++ b/.github/workflows/publish_image.yml @@ -1,3 +1,4 @@ +--- name: Build and push images to DockerHub on: diff --git a/.github/workflows/sphinx-build.yml b/.github/workflows/sphinx-build.yml new file mode 100644 index 0000000..807b8d9 --- /dev/null +++ b/.github/workflows/sphinx-build.yml @@ -0,0 +1,35 @@ +name: Sphinx Documentation + +on: + push: + branches: + - main # Adjust this to your default branch if it's not 'main' + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install sphinx ghp-import + + - name: Build Sphinx documentation + run: | + cd docs + make html + + - name: Deploy to GitHub Pages + run: | + cd docs/_build/html + ghp-import -n -p -f . + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/sphinx-pages.yml b/.github/workflows/sphinx-pages.yml deleted file mode 100644 index 6b44649..0000000 --- a/.github/workflows/sphinx-pages.yml +++ /dev/null @@ -1,24 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: Build Sphinx pages - -# Controls when the action will run. -on: - # Triggers the workflow on push or pull request events but only for the master branch - push: - branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - build: - name: Push Sphinx Pages - runs-on: ubuntu-latest - steps: - - uses: tdviet/sphinx-pages@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - create_readme: true - diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..55d039f --- /dev/null +++ b/.pylintrc @@ -0,0 +1,569 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths= + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.10 + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear and the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins ignore-mixin- +# members is set to 'yes' +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=no + +# Signatures are removed from the similarity computation +ignore-signatures=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=150 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + vo, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=10 + +# Maximum number of attributes for a class (see R0902). +max-attributes=10 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=20 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..c1049bf --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/FEDCLOUD_CONFIG b/FEDCLOUD_CONFIG new file mode 100644 index 0000000..ff18f94 --- /dev/null +++ b/FEDCLOUD_CONFIG @@ -0,0 +1,25 @@ +_MIN_ACCESS_TOKEN_TIME: 30 +gocdb_public_url: https://goc.egi.eu/gocdbpi/public/ +gocdb_service_group: org.openstack.nova +jaro221: jaro221 +log_config_file: /home/jaro221/.config/fedcloud/logging.conf +log_file: /home/jaro221/.config/fedcloud/logs/fedcloud.log +log_level: DEBUG +min_access_token_time: 30 +mytoken: OOOOO +mytoken_server: https://mytoken.data.kit.edu +oidc_agent_account: egi +oidc_url: https://aai.egi.eu/auth/realms/egi +os_auth_type: v3oidcaccesstoken +os_identity_provider: egi.eu +os_protocol: openid +requests_cert_file: /home/jaro221/.config/fedcloud/cert/certs.pem +site: IISAS-FedCloud +site_dir: /home/jaro221/.config/fedcloud/site-config +site_list_url: https://raw.githubusercontent.com/tdviet/fedcloudclient/master/config/sites.yaml +vault_endpoint: https://vault.services.fedcloud.eu:8200 +vault_locker_mount_point: /v1/cubbyhole/ +vault_mount_point: /secrets/ +vault_role: '' +vault_salt: fedcloud_salt +vo: vo.access.egi.eu diff --git a/docs/.buildinfo b/docs/.buildinfo new file mode 100644 index 0000000..81c6db5 --- /dev/null +++ b/docs/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file records the configuration used when building these files. When it is not found, a full rebuild will be done. +config: 9322cea65c0f49c249d6ce618a461990 +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/docs/FAQ.rst b/docs/_sources/docs/FAQ.rst.txt similarity index 100% rename from docs/FAQ.rst rename to docs/_sources/docs/FAQ.rst.txt diff --git a/docs/cheat.rst b/docs/_sources/docs/cheat.rst.txt similarity index 96% rename from docs/cheat.rst rename to docs/_sources/docs/cheat.rst.txt index ea899fe..155d40b 100644 --- a/docs/cheat.rst +++ b/docs/_sources/docs/cheat.rst.txt @@ -240,9 +240,9 @@ Useful commands FEDCLOUD_MYTOKEN= # created on https://mytoken.data.kit.edu/ # Pass it to OpenStack - EGI_SITE=IISAS-FedCloud - EGI_VO=vo.access.egi.eu - fedcloud openstack server create --site $EGI_SITE --flavor --image --user-data user.txt --key-name testvm + FEDCLOUD_SITE=IISAS-FedCloud + FEDCLOUD_VO=vo.access.egi.eu + fedcloud openstack server create --flavor --image --user-data user.txt --key-name testvm # Once you log into the VM you can retrieve the "mytoken" with curl http://169.254.169.254/openstack/latest/user_data/ diff --git a/docs/development.rst b/docs/_sources/docs/development.rst.txt similarity index 100% rename from docs/development.rst rename to docs/_sources/docs/development.rst.txt diff --git a/docs/fedcloudclient.rst b/docs/_sources/docs/fedcloudclient.rst.txt similarity index 100% rename from docs/fedcloudclient.rst rename to docs/_sources/docs/fedcloudclient.rst.txt diff --git a/docs/_sources/docs/index.rst.txt b/docs/_sources/docs/index.rst.txt new file mode 100644 index 0000000..61387dd --- /dev/null +++ b/docs/_sources/docs/index.rst.txt @@ -0,0 +1,26 @@ +.. fedcloudclient documentation master file, created by + sphinx-quickstart on Wed May 14 12:59:25 2025. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +fedcloudclient documentation +============================ + +Add your content using ``reStructuredText`` syntax. See the +`reStructuredText `_ +documentation for details. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + intro + usage + FAQ + cheat + development + install + modules + quickstart + scripts \ No newline at end of file diff --git a/docs/_sources/docs/install.rst.txt b/docs/_sources/docs/install.rst.txt new file mode 100644 index 0000000..55033d2 --- /dev/null +++ b/docs/_sources/docs/install.rst.txt @@ -0,0 +1,86 @@ +Installation +============ + +Installing FedCloud client with pip +*********************************** + +Simply use the following **pip3** command (should be done without root privileges). + +:: + + $ pip3 install -U fedcloudclient + +Installation Notes +------------------ + +Installing the latest version of the **FedCloud client** package using ``pip3`` will also install all required dependencies (such as **openstackclient**). It will create the executable files **``fedcloud``** and **``openstack``**, placing them in the appropriate directory based on your Python environment: + +* ``$VIRTUAL_ENV/bin`` – when using ``pip3`` inside a Python virtual environment, +* ``~/.local/bin`` – when installing with ``pip3 --user``, +* ``/usr/local/bin`` – when installing system-wide with ``pip3`` as root. + +.. note:: + + If you install the package with the ``--user`` option, make sure that ``~/.local/bin`` is included in your ``$PATH``. + +Verifying the Installation +-------------------------- + +To check if the installation was successful, run the following command:: + +:: + + $ fedcloud --help + + +This will show output:: + +:: + Usage: fedcloud [OPTIONS] COMMAND [ARGS]... + + CLI main function. Intentionally empty + + Options: + --version Show the version and exit. + --help Show this message and exit. + + Commands: + ec3 EC3 cluster provisioning + endpoint Obtain endpoint details and scoped tokens + openstack Execute OpenStack commands on site and VO + openstack-int Interactive OpenStack client on site and VO + secret Commands for accessing secret objects + select Select resources according to specification + site Obtain site configurations + token Get details of access token + + + +Installing EGI Core Trust Anchor certificates +********************************************* + +Some sites use certificates issued by national certificate authorities that are not included in the default +OS distribution. If you receive error message *"SSL exception connecting to https:// ..."*, +follow `instructions `_ +for installing EGI Core Trust Anchor certificates and add them to the certificate bundle of Python requests. For quick +test in virtual environment, just execute the following commands. See this +`README.md `_ for more details. + +:: + + $ wget https://raw.githubusercontent.com/tdviet/python-requests-bundle-certs/main/scripts/install_certs.sh + $ bash install_certs.sh + +Using FedCloud client via Docker container +****************************************** + +You can use Docker container for testing **FedCloud client** without installation. EGI Core Trust Anchor certificates +and site configurations are preinstalled. + +:: + + $ sudo docker pull tdviet/fedcloudclient + $ sudo docker run -it tdviet/fedcloudclient bash + + + diff --git a/docs/intro.rst b/docs/_sources/docs/intro.rst.txt similarity index 95% rename from docs/intro.rst rename to docs/_sources/docs/intro.rst.txt index 18bbe59..cd30666 100644 --- a/docs/intro.rst +++ b/docs/_sources/docs/intro.rst.txt @@ -30,7 +30,9 @@ The most notable features of FedCloud client are following: or as a `Python library `_ for programming FedCloud services. -Six modules are included: +The following modules are included: + +* **fedcloudclient.conf** for handling fedcloudclient configuration, * **fedcloudclient.checkin** for operation with EGI Check-in like getting tokens, diff --git a/docs/modules.rst b/docs/_sources/docs/modules.rst.txt similarity index 100% rename from docs/modules.rst rename to docs/_sources/docs/modules.rst.txt diff --git a/docs/quickstart.rst b/docs/_sources/docs/quickstart.rst.txt similarity index 98% rename from docs/quickstart.rst rename to docs/_sources/docs/quickstart.rst.txt index 3d10d70..2f08113 100644 --- a/docs/quickstart.rst +++ b/docs/_sources/docs/quickstart.rst.txt @@ -28,7 +28,7 @@ or use Docker container: :: - $ export OIDC_ACCESS_TOKEN= + $ export FEDCLOUD_OIDC_ACCESS_TOKEN= Basic usages ************ diff --git a/docs/scripts.rst b/docs/_sources/docs/scripts.rst.txt similarity index 100% rename from docs/scripts.rst rename to docs/_sources/docs/scripts.rst.txt diff --git a/docs/_sources/docs/usage.rst.txt b/docs/_sources/docs/usage.rst.txt new file mode 100644 index 0000000..4c5a0b0 --- /dev/null +++ b/docs/_sources/docs/usage.rst.txt @@ -0,0 +1,611 @@ +Usage +===== + +**FedCloud client** has the following main groups of commands: + +* **"fedcloud config"** for handling fedcloudclient configuration, + +* **"fedcloud token"** for interactions with EGI Check-in and access tokens, + +* **"fedcloud endpoint"** for interactions with GOCDB (and site endpoints according to GOCDB), + +* **"fedcloud site"** for manipulations with site configurations, + +* **"fedcloud openstack"** or **"fedcloud openstack-int"** for performing OpenStack commands on sites, + +* **"fedcloud secret"** for accessing secrets in + `Secret management service `_, + +* **"fedcloud ec3"** as helper commands for deploying EC3. + + + +Authentication Options +====================== + +**FedCloud** commands require access tokens for authentication. Users have multiple options for providing these tokens: + +- **Direct access token**: Use the ``--oidc-access-token`` option to provide an access token directly. You can retrieve this token from the environment variable ``FEDCLOUD_OIDC_ACCESS_TOKEN``, or pass it explicitly, e.g.: + + ``fedcloud token check --oidc-access-token `` + +- **OIDC agent**: Use the ``--oidc-agent-account`` option to integrate with `oidc-agent `_. For example, check token validity with: + + ``fedcloud token check --oidc-agent-account `` + + To use this method, follow the instructions at `oidc-agent for EGI `_ to register a client, then pass the client (account) name to the FedCloud client. + +- **Mytoken**: Use the ``--mytoken`` option to authenticate with a token from the `Mytoken service `_. To check token validity: + + ``fedcloud token check --mytoken `` + + When creating a Mytoken, ensure you select **"Allows obtaining OpenID Connect Access Tokens"**. You may also use the ``--mytoken-server`` option to authenticate with a specific Mytoken server. + +Alternatively, you can obtain tokens using the `EGI Check-in Token Portal `_, which provides all necessary information for EGI Check-in users. + +In addition to command-line options, environment variables can be used for passing tokens, as summarized in the table below (not shown here). + +By default, the protocol used is ``openid``. This can be changed using the ``--os-protocol`` option. Note that some sites may have a fixed protocol defined in their site configuration (e.g., ``oidc`` for INFN-CLOUD-BARI). +------- + +Configuration +************* + +Display the current configuration of *fedcloud* with: + +:: + + $ fedcloud config show + +This will show a list of configuration parameters: + ++----------------------------+------------------------------------------------------------------------------------+ +| Parameter | Default value | ++============================+====================================================================================+ +| site | IISAS-FedCloud | ++----------------------------+------------------------------------------------------------------------------------+ +| vo | vo.access.egi.eu | ++----------------------------+------------------------------------------------------------------------------------+ +| site_list_url | https://raw.githubusercontent.com/tdviet/fedcloudclient/master/config/sites.yaml | ++----------------------------+------------------------------------------------------------------------------------+ +| site_dir | ${HOME}/.config/fedcloud/site-config | ++----------------------------+------------------------------------------------------------------------------------+ +| oidc_url | https://aai.egi.eu/auth/realms/egi | ++----------------------------+------------------------------------------------------------------------------------+ +| gocdb_public_url | https://goc.egi.eu/gocdbpi/public/ | ++----------------------------+------------------------------------------------------------------------------------+ +| gocdb_service_group | org.openstack.nova | ++----------------------------+------------------------------------------------------------------------------------+ +| vault_endpoint | https://vault.services.fedcloud.eu:8200 | ++----------------------------+------------------------------------------------------------------------------------+ +| vault_role | | ++----------------------------+------------------------------------------------------------------------------------+ +| vault_mount_point | /secrets/ | ++----------------------------+------------------------------------------------------------------------------------+ +| vault_locker_mount_point | /v1/cubbyhole/ | ++----------------------------+------------------------------------------------------------------------------------+ +| vault_salt | fedcloud_salt | ++----------------------------+------------------------------------------------------------------------------------+ +| log_file | ${HOME}/.config/fedcloud/logs/fedcloud.log | ++----------------------------+------------------------------------------------------------------------------------+ +| log_level | DEBUG | ++----------------------------+------------------------------------------------------------------------------------+ +| log_config_file | ${HOME}/.config/fedcloud/logging.conf | ++----------------------------+------------------------------------------------------------------------------------+ +| requests_cert_file | ${HOME}/.config/fedcloud/cert/certs.pem | ++----------------------------+------------------------------------------------------------------------------------+ +| oidc_agent_account | egi | ++----------------------------+------------------------------------------------------------------------------------+ +| min_access_token_time | 30 | ++----------------------------+------------------------------------------------------------------------------------+ +| mytoken_server | https://mytoken.data.kit.edu | ++----------------------------+------------------------------------------------------------------------------------+ +| os_protocol | openid | ++----------------------------+------------------------------------------------------------------------------------+ +| os_auth_type | v3oidcaccesstoken | ++----------------------------+------------------------------------------------------------------------------------+ +| os_identity_provider | egi.eu | ++----------------------------+------------------------------------------------------------------------------------+ + + +The **FedCloud client** supports multiple types of configuration: + +- **Default settings** – accessible using **``DEFAULT_SETTINGS``**, these are the built-in default values. +- **Local environment settings** – custom configuration values defined in the environment and loaded via **``env_config``**. +- **Saved configuration settings** – user-defined settings stored in a JSON file, accessible via **``saved_config``**. + +For example, to print the environment configuration, use the following command:: + + $ fedcloud config show --source env_config + + +This command shows, for instance, the following output: + +:: + parameter value + ------------------ ------- + oidc_agent_account jaro221 + + +The *fedcloud* configuration can be saved to a file using the following command + +:: + $ fedcloud config create + +By default, the configuration file is saved to **${HOME}/.config/fedcloud/config.yaml**, +but this location can be changed using the ``--config`` option. For example: + +:: + + $ fedcloud config create --config-file /path/to/file.yaml + + +Using Environment Variables and Configuration Priorities +-------------------------------------------------------- + +It is also possible to use the *FEDCLOUD_CONFIG_FILE* environment variable instead of the ``--config`` option in the command line. +This allows users to manage and switch between multiple configuration files—one per project—each with its own settings. + +The *fedcloud* client supports configuration from multiple sources, in the following order of priority (highest to lowest): + +#. **Command-line options** – override all other settings. + Example: ``--site IISAS-FedCloud`` + +#. **Environment variables** – must begin with the prefix ``FEDCLOUD_``. + Example: ``FEDCLOUD_SITE=IISAS-FedCloud`` + +#. **Configuration file** – typically stored as ``config.yaml``. + Example: the ``site`` setting in ``config.yaml`` + +#. **Default configuration** – hardcoded defaults (lowest priority). + See the `source code <../fedcloudclient/conf.py#L16>`_ for details. + +The priority order is important: +default values are overridden by the configuration file, +which is overridden by environment variables, +which are in turn overridden by command-line options. + +For example, the default configuration includes: + +- ``site = IISAS-FedCloud`` +- ``vo = vo.access.egi.eu`` + +These values can be changed using any of the higher-priority methods. For example: + +:: + + $ fedcloud openstack --vo training.egi.eu --site IFCA-LCG2 server list + +or + +:: + + $ export FEDCLOUD_VO=training.egi.eu + $ export FEDCLOUD_SITE=IFCA-LCG2 + $ fedcloud openstack server list + + +Consistent Parameter Naming +--------------------------- + +Note the consistent naming convention for configuration parameters across different sources. For example, the same parameter is represented as: + +* ``--oidc-agent-account`` in the **command-line** +* ``FEDCLOUD_OIDC_AGENT_ACCOUNT`` as an **environment variable** +* ``oidc_agent_account`` in the **configuration file** + +All configuration parameters follow this consistent mapping across command-line options, environment variables, and configuration files. + +Additional Configurable Parameters +---------------------------------- + +In addition to ``oidc_agent_account``, the following parameters can also be configured in the same way: + +* ``site`` – the OpenStack site to target +* ``vo`` – the Virtual Organisation (VO) +* ``check_in_url`` – the EGI Check-in OIDC endpoint +* ``client_id`` – the OIDC client ID +* ``scopes`` – requested OIDC scopes +* ``access_token`` – manually provided access token +* ``output_format`` – format of output, e.g., ``table``, ``json``, or ``yaml`` + +These parameters can be specified via: + +- command-line options (e.g., ``--site``, ``--vo``), +- environment variables (e.g., ``FEDCLOUD_SITE``, ``FEDCLOUD_VO``), or +- configuration files (e.g., ``site: IISAS-FedCloud`` in ``config.yaml``). + +This design allows flexible and convenient configuration for various usage scenarios. + ++------------------------------+-------------------------+ +| Environment variable | Command-line option | ++==============================+=========================+ +| FEDCLOUD_OIDC_ACCESS_TOKEN | --oidc-access-token | ++------------------------------+-------------------------+ +| FEDCLOUD_MYTOKEN | --mytoken | ++------------------------------+-------------------------+ +| FEDCLOUD_OIDC_AGENT_ACCOUNT | --oidc-agent-account | ++------------------------------+-------------------------+ + +For convenience, it is recommended to set transient parameters—such as access tokens—via **environment variables**. +This simplifies the usage of *fedcloud* commands by avoiding the need to specify these parameters on the command line each time. + + +Shell completion +**************** + +Shell completion for *fedcloud* command in *bash* can be activated by executing the following command: + +:: + + $ eval "$(_FEDCLOUD_COMPLETE=bash_source fedcloud)" + +The command above may affect responsiveness of the shell. For long work, it is recommended to copy the +`fedcloud_bash_completion.sh script +`_ to a local file, and +source it from ~/.bashrc. Refer `Click documentation +`_ for a long explanation. + +After enabling shell completion, press twice for shell completion: + +:: + + $ fedcloud site + env list save-config show show-project-id + + +fedcloud --help command +*********************** + +* **"fedcloud --help"** command will print help message. When using it in combination with other + commands, e.g. **"fedcloud token --help"**, **"fedcloud token check --hep"**, it will print list of options for the + corresponding commands + +:: + + $ fedcloud --help + Usage: fedcloud [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Commands: + config Managing fedcloud configurations + endpoint Obtain endpoint details and scoped tokens + openstack Execute OpenStack commands on site and VO + openstack-int Interactive OpenStack client on site and VO + secret Commands for accessing secret objects + select Select resources according to specification + site Obtain site configurations + token Get details of access token + + +fedcloud token commands +*********************** + +* **``fedcloud token check``** – Checks the expiration time of the configured access token, + allowing users to determine whether it needs to be refreshed. + +As mentioned earlier, the access token can be provided via the environment variable ``FEDCLOUD_OIDC_ACCESS_TOKEN``. +For this reason, the ``--oidc-access-token`` option is not shown in all examples below, even though it may be required if the token is not set via environment variables. + + +:: + + $ fedcloud token check + +Output is shown as: +:: + Access token is valid to 2021-01-02 01:25:39 UTC + Access token expires in 3571 seconds + + +* **"fedcloud token list-vos"** : Print the list of VO memberships according to EGI Check-in + +:: + + $ fedcloud token list-vos + +Sample output: +:: + eosc-synergy.eu + fedcloud.egi.eu + training.egi.eu + +* **"fedcloud token issue"** : Print the access_token + +:: + + $ fedcloud token issue + +Sample output: +:: + egwergwregrwegreg... + +fedcloud endpoint commands +************************** + +**"fedcloud endpoint"** commands are complementary part of the **"fedcloud site"** commands. Instead of using site +configurations defined in files saved in GitHub repository or local disk, the commands try to get site information +directly from GOCDB (Grid Operations Configuration Management Database) https://goc.egi.eu/ or make probe test on sites + +* **"fedcloud endpoint list"** : List of endpoints of sites defined in GOCDB. + +:: + + $ fedcloud endpoint list + +Sample output: + +:: + Site type URL + ------------------ ------------------ ------------------------------------------------ + IFCA-LCG2 org.openstack.nova https://api.cloud.ifca.es:5000/v3/ + IN2P3-IRES org.openstack.nova https://sbgcloud.in2p3.fr:5000/v3 + ... + + +* **"fedcloud endpoint projects --site --oidc-access-token "** : List of projects to which the owner + of the access token has access at the given site + +:: + + $ fedcloud endpoint projects --site IFCA-LCG2 + id Name enabled site + -------------------------------- -------------------------- --------- --------- + 2a7e2cd4b6dc4e609dd934964c1715c6 VO:demo.fedcloud.egi.eu True IFCA-LCG2 + 3b9754ad8c6046b4aec43ec21abe7d8c VO:eosc-synergy.eu True IFCA-LCG2 + ... + +If the site is set to *ALL_SITES*, or the argument *-a* is used, the command will show accessible projects from all sites of the EGI Federated Cloud. + + +* **"fedcloud endpoint vos --site --oidc-access-token "** : List of Virtual Organisations (VOs) + to which the owner of the access token has access at the given site + +:: + + $ fedcloud endpoint vos --site IFCA-LCG2 + VO id Project name enabled site + ---------------- -------------------------------- ------------------- --------- --------- + vo.access.egi.eu 233f045cb1ff46842a15ebb33af69460 VO:vo.access.egi.eu True IFCA-LCG2 + training.egi.eu d340308880134d04294097524eace710 VO:training.egi.eu True IFCA-LCG2 + ... + +If the site is set to *ALL_SITES*, or the argument *-a* is used, the command will show accessible VOs from all sites of the EGI Federated Cloud. + +:: + + $ fedcloud endpoint vos -a + VO id Project name enabled site + ------------------- -------------------------------- ------------------- --------- ----------------- + vo.access.egi.eu 233f045cb1ff46842a15ebb33af69460 VO:vo.access.egi.eu True IFCA-LCG2 + training.egi.eu d340308880134d04294097524eace710 VO:training.egi.eu True IFCA-LCG2 + vo.access.egi.eu 7101022b9ae74ed9ac1a574497279499 EGI_access True IN2P3-IRES + vo.access.egi.eu 5bbdb5c1e0b2bcbac29904f4ac22dcaa vo_access_egi_eu True UNIV-LILLE + vo.access.egi.eu 4cab325ca8c2495bf2d4e8f230bcd51a VO:vo.access.egi.eu True INFN-PADOVA-STACK + ... + + +* **"fedcloud endpoint token --site --project-id --oidc-access-token "** : Get + OpenStack keystone scoped token on the site for the project ID. + +:: + + $ fedcloud endpoint token --site IFCA-LCG2 --project-id 3b9754ad8c6046b4aec43ec21abe7d8c + export FEDCLOUD_OS_TOKEN="gAAAAA..." + + +* **"fedcloud endpoint env --site --project-id --oidc-access-token "** : Print + environment variables for working with the project ID on the site. + +:: + + $ fedcloud endpoint env --site IFCA-LCG2 --project-id 3b9754ad8c6046b4aec43ec21abe7d8c + # environment for IFCA-LCG2 + export FEDCLOUD_OS_AUTH_URL="https://api.cloud.ifca.es:5000/v3/" + export FEDCLOUD_OS_AUTH_TYPE="v3oidcaccesstoken" + export FEDCLOUD_OS_IDENTITY_PROVIDER="egi.eu" + export FEDCLOUD_OS_PROTOCOL="openid" + export FEDCLOUD_OS_ACCESS_TOKEN="..." + + +fedcloud site commands +********************** + +**"fedcloud site"** commands will read site configurations and manipulate with them. If the local site configurations +exist at *~/.config/fedcloud/site-config/*, **fedcloud** will read them from there, otherwise the commands will read +from `GitHub repository `_. + +By default, **fedcloud** does not save anything on local disk, users have to save the site configuration to local disk +explicitly via **"fedcloud site save-config"** command. The advantage of having local +site configurations, beside faster loading, is to give users ability to make customizations, e.g. add additional VOs, +remove sites they do not have access, and so on. + +* **"fedcloud site save-config"** : Read the default site configurations from GitHub + and save them to *~/.config/fedcloud/site-config/* local directory. The command will overwrite existing site configurations + in the local directory. + +:: + + $ fedcloud site save-config + Saving site configs to directory /home/viet/.config/fedcloud/site-config/ + + +After saving site configurations, users can edit and customize them, e.g. remove inaccessible sites, add new +VOs and so on. + +* **"fedcloud site list"** : List of existing sites in the site configurations + +:: + + $ fedcloud site list + 100IT + BIFI + CESGA + ... + + +* **"fedcloud site list --vo "** : List all sites supporting a Virtual Organization + +:: + + $ fedcloud site vo-list --vo vo.access.egi.eu + BIFI + CENI + CESGA-CLOUD + ... + + +* **"fedcloud site show --site "** : Show configuration of the corresponding site. + +:: + + $ fedcloud site show --site IISAS-FedCloud + endpoint: https://cloud.ui.savba.sk:5000/v3/ + gocdb: IISAS-FedCloud + vos: + - auth: + project_id: a22bbffb007745b2934bf308b0a4d186 + name: covid19.eosc-synergy.eu + - auth: + project_id: 51f736d36ce34b9ebdf196cfcabd24ee + name: eosc-synergy.eu + + +* **"fedcloud site show-project-id --site --vo "**: show the project ID of the VO on the site. + +:: + + $ fedcloud site show-project-id --site IISAS-FedCloud --vo eosc-synergy.eu + export FEDCLOUD_OS_AUTH_URL="https://cloud.ui.savba.sk:5000/v3/" + export FEDCLOUD_OS_PROJECT_ID="51f736d36ce34b9ebdf196cfcabd24ee" + + +* **"fedcloud site env --site --vo "**: set OpenStack environment variable for the VO on the site. + +:: + + $ fedcloud site env --site IISAS-FedCloud --vo eosc-synergy.eu + export FEDCLOUD_OS_AUTH_URL="https://cloud.ui.savba.sk:5000/v3/" + export FEDCLOUD_OS_AUTH_TYPE="v3oidcaccesstoken" + export FEDCLOUD_OS_IDENTITY_PROVIDER="egi.eu" + export FEDCLOUD_OS_PROTOCOL="openid" + export FEDCLOUD_OS_PROJECT_ID="51f736d36ce34b9ebdf196cfcabd24ee" + # Remember to set OS_ACCESS_TOKEN, e.g. : + # export FEDCLOUD_OS_ACCESS_TOKEN=`oidc-token egi` + + +The main differences between *"fedcloud endpoint env"* and *"fedcloud site env"* commands are that the second command +needs VO name as input parameter instead of project ID. The command may set also environment variable OS_ACCESS_TOKEN, +if access token is provided, otherwise it will print notification. + + +fedcloud select commands +*************************** + +* **"fedcloud select flavor --site --vo --oidc-access-token --flavor-specs "** : + Select flavor according to the specification in *flavor-specs*. The specifications may be repeated, + e.g. *--flavor-specs "VCPUs==2" --flavor-specs "RAM>=2048"*, or may be joined, e.g. + *--flavor-specs "VCPUs==2 & Disk>10"*. For frequently used specs, short-option alternatives are available, e.g. + *--vcpus 2* is equivalent to *--flavor-specs "VCPUs==2"*. The output is sorted, flavors using less resources + (in the order: GPUs, CPUs, RAM, Disk) are placed on the first places. Users can choose to print only the best-matched + flavor with *--output-format first* (suitable for scripting) or the full list of all matched flavors in list/YAML/JSON + format. + +:: + + $ fedcloud select flavor --site IISAS-FedCloud --vo vo.access.egi.eu --flavor-specs "RAM>=2096" --flavor-specs "Disk > 10" --output-format list + m1.medium + m1.large + m1.xlarge + m1.huge + g1.c08r30-K20m + g1.c16r60-2xK20m + + +* **"fedcloud select image --site --vo --oidc-access-token --image-specs "** : + Select image according to the specification in *image-specs*. The specifications may be repeated, + e.g. *--image-specs "Name=~Ubuntu" --image-specs "Name=~'20.04'"*. The output is sorted, newest images + are placed on the first places. Users can choose to print only the best-matched + image with *--output-format first* (suitable for scripting) or the full list of all matched images in list/YAML/JSON + format. + +:: + + $ fedcloud select image --site INFN-CATANIA-STACK --vo training.egi.eu --image-specs "Name =~ Ubuntu" --output-format list + TRAINING.EGI.EU Image for EGI Docker [Ubuntu/18.04/VirtualBox] + TRAINING.EGI.EU Image for EGI Ubuntu 20.04 [Ubuntu/20.04/VirtualBox] + + +* **"fedcloud select network --site --vo --oidc-access-token --network-specs "** : + Select network according to the specification in *network-specs*. User can choose to select only public or private + network, or both (default). The output is sorted in the order: public, shared, + private. Users can choose to print only the best-matched network with *--output-format first* + (suitable for scripting) or the full list of all matched networks in list/YAML/JSON format. + +:: + + $ fedcloud select network --site IISAS-FedCloud --vo training.egi.eu --network-specs default --output-format list + public-network + private-network + + +fedcloud openstack commands +*************************** + +* **"fedcloud openstack --site --vo --oidc-access-token "** : Execute an + OpenStack command on the site and VO. Examples of OpenStack commands are *"image list"*, *"server list"* and can be used + with additional options for the commands, e.g. *"image list --long"*, *"server list --format json"*. The list of all + OpenStack commands, and their parameters/usages are available + `here `_. + +:: + + $ fedcloud openstack image list --site IISAS-FedCloud --vo eosc-synergy.eu + Site: IISAS-FedCloud, VO: eosc-synergy.eu + +--------------------------------------+-------------------------------------------------+--------+ + | ID | Name | Status | + +--------------------------------------+-------------------------------------------------+--------+ + | 862d4ede-6a11-4227-8388-c94141a5dace | Image for EGI CentOS 7 [CentOS/7/VirtualBox] | active | + ... + + +If the site is *ALL_SITES*, the OpenStack command will be executed on all sites in EGI Federated Cloud. + +* **"fedcloud openstack-int --site --vo --oidc-access-token "** : Call OpenStack client without + command, so users can work with OpenStack site in interactive mode. This is useful when users need to perform multiple + commands successively. For example, users may need get list of images, list of flavors, list of networks before + creating a VM. OIDC authentication is done only once at the beginning, then the keystone token is cached and will + be used for successive commands without authentication via CheckIn again. + +:: + + $ fedcloud openstack-int --site IISAS-FedCloud --vo eosc-synergy.eu + (openstack) image list + +--------------------------------------+-------------------------------------------------+--------+ + | ID | Name | Status | + +--------------------------------------+-------------------------------------------------+--------+ + | 862d4ede-6a11-4227-8388-c94141a5dace | Image for EGI CentOS 7 [CentOS/7/VirtualBox] | active | + ... + (openstack) flavor list + +--------------------------------------+-----------+-------+------+-----------+-------+-----------+ + | ID | Name | RAM | Disk | Ephemeral | VCPUs | Is Public | + +--------------------------------------+-----------+-------+------+-----------+-------+-----------+ + | 5bd8397c-b97f-462d-9d2b-5b533844996c | m1.small | 2048 | 10 | 0 | 1 | True | + | df25f80f-ed19-4e0b-805e-d34620ba0334 | m1.medium | 4096 | 40 | 0 | 2 | True | + ... + (openstack) + + +fedcloud config commands +*************************** +* **"fedcloud config --config-file create"** : Create default configuration file in default location for configuration file + + + +fedcloud secret commands +*************************** + +The **"fedcloud secret"** commands are described in details in the documentation of the +`Secret management service `_. diff --git a/docs/_sources/index.rst.txt b/docs/_sources/index.rst.txt new file mode 100644 index 0000000..c79b1a7 --- /dev/null +++ b/docs/_sources/index.rst.txt @@ -0,0 +1,17 @@ +.. fedcloudclient documentation master file, created by + sphinx-quickstart on Thu May 15 09:19:32 2025. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +fedcloudclient documentation +============================ + +Add your content using ``reStructuredText`` syntax. See the +`reStructuredText `_ +documentation for details. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + diff --git a/docs/_static/alabaster.css b/docs/_static/alabaster.css new file mode 100644 index 0000000..7e75bf8 --- /dev/null +++ b/docs/_static/alabaster.css @@ -0,0 +1,663 @@ +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: Georgia, serif; + font-size: 17px; + background-color: #fff; + color: #000; + margin: 0; + padding: 0; +} + + +div.document { + width: 940px; + margin: 30px auto 0 auto; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 220px; +} + +div.sphinxsidebar { + width: 220px; + font-size: 14px; + line-height: 1.5; +} + +hr { + border: 1px solid #B1B4B6; +} + +div.body { + background-color: #fff; + color: #3E4349; + padding: 0 30px 0 30px; +} + +div.body > .section { + text-align: left; +} + +div.footer { + width: 940px; + margin: 20px auto 30px auto; + font-size: 14px; + color: #888; + text-align: right; +} + +div.footer a { + color: #888; +} + +p.caption { + font-family: inherit; + font-size: inherit; +} + + +div.relations { + display: none; +} + + +div.sphinxsidebar { + max-height: 100%; + overflow-y: auto; +} + +div.sphinxsidebar a { + color: #444; + text-decoration: none; + border-bottom: 1px dotted #999; +} + +div.sphinxsidebar a:hover { + border-bottom: 1px solid #999; +} + +div.sphinxsidebarwrapper { + padding: 18px 10px; +} + +div.sphinxsidebarwrapper p.logo { + padding: 0; + margin: -10px 0 0 0px; + text-align: center; +} + +div.sphinxsidebarwrapper h1.logo { + margin-top: -10px; + text-align: center; + margin-bottom: 5px; + text-align: left; +} + +div.sphinxsidebarwrapper h1.logo-name { + margin-top: 0px; +} + +div.sphinxsidebarwrapper p.blurb { + margin-top: 0; + font-style: normal; +} + +div.sphinxsidebar h3, +div.sphinxsidebar h4 { + font-family: Georgia, serif; + color: #444; + font-size: 24px; + font-weight: normal; + margin: 0 0 5px 0; + padding: 0; +} + +div.sphinxsidebar h4 { + font-size: 20px; +} + +div.sphinxsidebar h3 a { + color: #444; +} + +div.sphinxsidebar p.logo a, +div.sphinxsidebar h3 a, +div.sphinxsidebar p.logo a:hover, +div.sphinxsidebar h3 a:hover { + border: none; +} + +div.sphinxsidebar p { + color: #555; + margin: 10px 0; +} + +div.sphinxsidebar ul { + margin: 10px 0; + padding: 0; + color: #000; +} + +div.sphinxsidebar ul li.toctree-l1 > a { + font-size: 120%; +} + +div.sphinxsidebar ul li.toctree-l2 > a { + font-size: 110%; +} + +div.sphinxsidebar input { + border: 1px solid #CCC; + font-family: Georgia, serif; + font-size: 1em; +} + +div.sphinxsidebar #searchbox { + margin: 1em 0; +} + +div.sphinxsidebar .search > div { + display: table-cell; +} + +div.sphinxsidebar hr { + border: none; + height: 1px; + color: #AAA; + background: #AAA; + + text-align: left; + margin-left: 0; + width: 50%; +} + +div.sphinxsidebar .badge { + border-bottom: none; +} + +div.sphinxsidebar .badge:hover { + border-bottom: none; +} + +/* To address an issue with donation coming after search */ +div.sphinxsidebar h3.donation { + margin-top: 10px; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #004B6B; + text-decoration: underline; +} + +a:hover { + color: #6D4100; + text-decoration: underline; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: Georgia, serif; + font-weight: normal; + margin: 30px 0px 10px 0px; + padding: 0; +} + +div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } +div.body h2 { font-size: 180%; } +div.body h3 { font-size: 150%; } +div.body h4 { font-size: 130%; } +div.body h5 { font-size: 100%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: #DDD; + padding: 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + color: #444; + background: #EAEAEA; +} + +div.body p, div.body dd, div.body li { + line-height: 1.4em; +} + +div.admonition { + margin: 20px 0px; + padding: 10px 30px; + background-color: #EEE; + border: 1px solid #CCC; +} + +div.admonition tt.xref, div.admonition code.xref, div.admonition a tt { + background-color: #FBFBFB; + border-bottom: 1px solid #fafafa; +} + +div.admonition p.admonition-title { + font-family: Georgia, serif; + font-weight: normal; + font-size: 24px; + margin: 0 0 10px 0; + padding: 0; + line-height: 1; +} + +div.admonition p.last { + margin-bottom: 0; +} + +dt:target, .highlight { + background: #FAF3E8; +} + +div.warning { + background-color: #FCC; + border: 1px solid #FAA; +} + +div.danger { + background-color: #FCC; + border: 1px solid #FAA; + -moz-box-shadow: 2px 2px 4px #D52C2C; + -webkit-box-shadow: 2px 2px 4px #D52C2C; + box-shadow: 2px 2px 4px #D52C2C; +} + +div.error { + background-color: #FCC; + border: 1px solid #FAA; + -moz-box-shadow: 2px 2px 4px #D52C2C; + -webkit-box-shadow: 2px 2px 4px #D52C2C; + box-shadow: 2px 2px 4px #D52C2C; +} + +div.caution { + background-color: #FCC; + border: 1px solid #FAA; +} + +div.attention { + background-color: #FCC; + border: 1px solid #FAA; +} + +div.important { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.note { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.tip { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.hint { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.seealso { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.topic { + background-color: #EEE; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre, tt, code { + font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace; + font-size: 0.9em; +} + +.hll { + background-color: #FFC; + margin: 0 -12px; + padding: 0 12px; + display: block; +} + +img.screenshot { +} + +tt.descname, tt.descclassname, code.descname, code.descclassname { + font-size: 0.95em; +} + +tt.descname, code.descname { + padding-right: 0.08em; +} + +img.screenshot { + -moz-box-shadow: 2px 2px 4px #EEE; + -webkit-box-shadow: 2px 2px 4px #EEE; + box-shadow: 2px 2px 4px #EEE; +} + +table.docutils { + border: 1px solid #888; + -moz-box-shadow: 2px 2px 4px #EEE; + -webkit-box-shadow: 2px 2px 4px #EEE; + box-shadow: 2px 2px 4px #EEE; +} + +table.docutils td, table.docutils th { + border: 1px solid #888; + padding: 0.25em 0.7em; +} + +table.field-list, table.footnote { + border: none; + -moz-box-shadow: none; + -webkit-box-shadow: none; + box-shadow: none; +} + +table.footnote { + margin: 15px 0; + width: 100%; + border: 1px solid #EEE; + background: #FDFDFD; + font-size: 0.9em; +} + +table.footnote + table.footnote { + margin-top: -15px; + border-top: none; +} + +table.field-list th { + padding: 0 0.8em 0 0; +} + +table.field-list td { + padding: 0; +} + +table.field-list p { + margin-bottom: 0.8em; +} + +/* Cloned from + * https://github.com/sphinx-doc/sphinx/commit/ef60dbfce09286b20b7385333d63a60321784e68 + */ +.field-name { + -moz-hyphens: manual; + -ms-hyphens: manual; + -webkit-hyphens: manual; + hyphens: manual; +} + +table.footnote td.label { + width: .1px; + padding: 0.3em 0 0.3em 0.5em; +} + +table.footnote td { + padding: 0.3em 0.5em; +} + +dl { + margin-left: 0; + margin-right: 0; + margin-top: 0; + padding: 0; +} + +dl dd { + margin-left: 30px; +} + +blockquote { + margin: 0 0 0 30px; + padding: 0; +} + +ul, ol { + /* Matches the 30px from the narrow-screen "li > ul" selector below */ + margin: 10px 0 10px 30px; + padding: 0; +} + +pre { + background: unset; + padding: 7px 30px; + margin: 15px 0px; + line-height: 1.3em; +} + +div.viewcode-block:target { + background: #ffd; +} + +dl pre, blockquote pre, li pre { + margin-left: 0; + padding-left: 30px; +} + +tt, code { + background-color: #ecf0f3; + color: #222; + /* padding: 1px 2px; */ +} + +tt.xref, code.xref, a tt { + background-color: #FBFBFB; + border-bottom: 1px solid #fff; +} + +a.reference { + text-decoration: none; + border-bottom: 1px dotted #004B6B; +} + +a.reference:hover { + border-bottom: 1px solid #6D4100; +} + +/* Don't put an underline on images */ +a.image-reference, a.image-reference:hover { + border-bottom: none; +} + +a.footnote-reference { + text-decoration: none; + font-size: 0.7em; + vertical-align: top; + border-bottom: 1px dotted #004B6B; +} + +a.footnote-reference:hover { + border-bottom: 1px solid #6D4100; +} + +a:hover tt, a:hover code { + background: #EEE; +} + +@media screen and (max-width: 940px) { + + body { + margin: 0; + padding: 20px 30px; + } + + div.documentwrapper { + float: none; + background: #fff; + margin-left: 0; + margin-top: 0; + margin-right: 0; + margin-bottom: 0; + } + + div.sphinxsidebar { + display: block; + float: none; + width: unset; + margin: 50px -30px -20px -30px; + padding: 10px 20px; + background: #333; + color: #FFF; + } + + div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, + div.sphinxsidebar h3 a { + color: #fff; + } + + div.sphinxsidebar a { + color: #AAA; + } + + div.sphinxsidebar p.logo { + display: none; + } + + div.document { + width: 100%; + margin: 0; + } + + div.footer { + display: none; + } + + div.bodywrapper { + margin: 0; + } + + div.body { + min-height: 0; + min-width: auto; /* fixes width on small screens, breaks .hll */ + padding: 0; + } + + .hll { + /* "fixes" the breakage */ + width: max-content; + } + + .rtd_doc_footer { + display: none; + } + + .document { + width: auto; + } + + .footer { + width: auto; + } + + .github { + display: none; + } + + ul { + margin-left: 0; + } + + li > ul { + /* Matches the 30px from the "ul, ol" selector above */ + margin-left: 30px; + } +} + + +/* misc. */ + +.revsys-inline { + display: none!important; +} + +/* Hide ugly table cell borders in ..bibliography:: directive output */ +table.docutils.citation, table.docutils.citation td, table.docutils.citation th { + border: none; + /* Below needed in some edge cases; if not applied, bottom shadows appear */ + -moz-box-shadow: none; + -webkit-box-shadow: none; + box-shadow: none; +} + + +/* relbar */ + +.related { + line-height: 30px; + width: 100%; + font-size: 0.9rem; +} + +.related.top { + border-bottom: 1px solid #EEE; + margin-bottom: 20px; +} + +.related.bottom { + border-top: 1px solid #EEE; +} + +.related ul { + padding: 0; + margin: 0; + list-style: none; +} + +.related li { + display: inline; +} + +nav#rellinks { + float: right; +} + +nav#rellinks li+li:before { + content: "|"; +} + +nav#breadcrumbs li+li:before { + content: "\00BB"; +} + +/* Hide certain items when printing */ +@media print { + div.related { + display: none; + } +} + +img.github { + position: absolute; + top: 0; + border: 0; + right: 0; +} \ No newline at end of file diff --git a/docs/_static/basic.css b/docs/_static/basic.css new file mode 100644 index 0000000..d9846da --- /dev/null +++ b/docs/_static/basic.css @@ -0,0 +1,914 @@ +/* + * Sphinx stylesheet -- basic theme. + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +div.section::after { + display: block; + content: ''; + clear: left; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; + word-wrap: break-word; + overflow-wrap : break-word; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +div.sphinxsidebar #searchbox form.search { + overflow: hidden; +} + +div.sphinxsidebar #searchbox input[type="text"] { + float: left; + width: 80%; + padding: 0.25em; + box-sizing: border-box; +} + +div.sphinxsidebar #searchbox input[type="submit"] { + float: left; + width: 20%; + border-left: none; + padding: 0.25em; + box-sizing: border-box; +} + + +img { + border: 0; + max-width: 100%; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin-top: 10px; +} + +ul.search li { + padding: 5px 0; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li p.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; + margin-left: auto; + margin-right: auto; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable { + width: 100%; +} + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable ul { + margin-top: 0; + margin-bottom: 0; + list-style-type: none; +} + +table.indextable > tbody > tr > td > ul { + padding-left: 0em; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +div.modindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +div.genindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +/* -- domain module index --------------------------------------------------- */ + +table.modindextable td { + padding: 2px; + border-collapse: collapse; +} + +/* -- general body styles --------------------------------------------------- */ + +div.body { + min-width: inherit; + max-width: 800px; +} + +div.body p, div.body dd, div.body li, div.body blockquote { + -moz-hyphens: auto; + -ms-hyphens: auto; + -webkit-hyphens: auto; + hyphens: auto; +} + +a.headerlink { + visibility: hidden; +} + +a:visited { + color: #551A8B; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink, +caption:hover > a.headerlink, +p.caption:hover > a.headerlink, +div.code-block-caption:hover > a.headerlink { + visibility: visible; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.first { + margin-top: 0 !important; +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +img.align-left, figure.align-left, .figure.align-left, object.align-left { + clear: left; + float: left; + margin-right: 1em; +} + +img.align-right, figure.align-right, .figure.align-right, object.align-right { + clear: right; + float: right; + margin-left: 1em; +} + +img.align-center, figure.align-center, .figure.align-center, object.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +img.align-default, figure.align-default, .figure.align-default { + display: block; + margin-left: auto; + margin-right: auto; +} + +.align-left { + text-align: left; +} + +.align-center { + text-align: center; +} + +.align-default { + text-align: center; +} + +.align-right { + text-align: right; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar, +aside.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px; + background-color: #ffe; + width: 40%; + float: right; + clear: right; + overflow-x: auto; +} + +p.sidebar-title { + font-weight: bold; +} + +nav.contents, +aside.topic, +div.admonition, div.topic, blockquote { + clear: left; +} + +/* -- topics ---------------------------------------------------------------- */ + +nav.contents, +aside.topic, +div.topic { + border: 1px solid #ccc; + padding: 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- content of sidebars/topics/admonitions -------------------------------- */ + +div.sidebar > :last-child, +aside.sidebar > :last-child, +nav.contents > :last-child, +aside.topic > :last-child, +div.topic > :last-child, +div.admonition > :last-child { + margin-bottom: 0; +} + +div.sidebar::after, +aside.sidebar::after, +nav.contents::after, +aside.topic::after, +div.topic::after, +div.admonition::after, +blockquote::after { + display: block; + content: ''; + clear: both; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.docutils { + margin-top: 10px; + margin-bottom: 10px; + border: 0; + border-collapse: collapse; +} + +table.align-center { + margin-left: auto; + margin-right: auto; +} + +table.align-default { + margin-left: auto; + margin-right: auto; +} + +table caption span.caption-number { + font-style: italic; +} + +table caption span.caption-text { +} + +table.docutils td, table.docutils th { + padding: 1px 8px 1px 5px; + border-top: 0; + border-left: 0; + border-right: 0; + border-bottom: 1px solid #aaa; +} + +th { + text-align: left; + padding-right: 5px; +} + +table.citation { + border-left: solid 1px gray; + margin-left: 1px; +} + +table.citation td { + border-bottom: none; +} + +th > :first-child, +td > :first-child { + margin-top: 0px; +} + +th > :last-child, +td > :last-child { + margin-bottom: 0px; +} + +/* -- figures --------------------------------------------------------------- */ + +div.figure, figure { + margin: 0.5em; + padding: 0.5em; +} + +div.figure p.caption, figcaption { + padding: 0.3em; +} + +div.figure p.caption span.caption-number, +figcaption span.caption-number { + font-style: italic; +} + +div.figure p.caption span.caption-text, +figcaption span.caption-text { +} + +/* -- field list styles ----------------------------------------------------- */ + +table.field-list td, table.field-list th { + border: 0 !important; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.field-name { + -moz-hyphens: manual; + -ms-hyphens: manual; + -webkit-hyphens: manual; + hyphens: manual; +} + +/* -- hlist styles ---------------------------------------------------------- */ + +table.hlist { + margin: 1em 0; +} + +table.hlist td { + vertical-align: top; +} + +/* -- object description styles --------------------------------------------- */ + +.sig { + font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace; +} + +.sig-name, code.descname { + background-color: transparent; + font-weight: bold; +} + +.sig-name { + font-size: 1.1em; +} + +code.descname { + font-size: 1.2em; +} + +.sig-prename, code.descclassname { + background-color: transparent; +} + +.optional { + font-size: 1.3em; +} + +.sig-paren { + font-size: larger; +} + +.sig-param.n { + font-style: italic; +} + +/* C++ specific styling */ + +.sig-inline.c-texpr, +.sig-inline.cpp-texpr { + font-family: unset; +} + +.sig.c .k, .sig.c .kt, +.sig.cpp .k, .sig.cpp .kt { + color: #0033B3; +} + +.sig.c .m, +.sig.cpp .m { + color: #1750EB; +} + +.sig.c .s, .sig.c .sc, +.sig.cpp .s, .sig.cpp .sc { + color: #067D17; +} + + +/* -- other body styles ----------------------------------------------------- */ + +ol.arabic { + list-style: decimal; +} + +ol.loweralpha { + list-style: lower-alpha; +} + +ol.upperalpha { + list-style: upper-alpha; +} + +ol.lowerroman { + list-style: lower-roman; +} + +ol.upperroman { + list-style: upper-roman; +} + +:not(li) > ol > li:first-child > :first-child, +:not(li) > ul > li:first-child > :first-child { + margin-top: 0px; +} + +:not(li) > ol > li:last-child > :last-child, +:not(li) > ul > li:last-child > :last-child { + margin-bottom: 0px; +} + +ol.simple ol p, +ol.simple ul p, +ul.simple ol p, +ul.simple ul p { + margin-top: 0; +} + +ol.simple > li:not(:first-child) > p, +ul.simple > li:not(:first-child) > p { + margin-top: 0; +} + +ol.simple p, +ul.simple p { + margin-bottom: 0; +} + +aside.footnote > span, +div.citation > span { + float: left; +} +aside.footnote > span:last-of-type, +div.citation > span:last-of-type { + padding-right: 0.5em; +} +aside.footnote > p { + margin-left: 2em; +} +div.citation > p { + margin-left: 4em; +} +aside.footnote > p:last-of-type, +div.citation > p:last-of-type { + margin-bottom: 0em; +} +aside.footnote > p:last-of-type:after, +div.citation > p:last-of-type:after { + content: ""; + clear: both; +} + +dl.field-list { + display: grid; + grid-template-columns: fit-content(30%) auto; +} + +dl.field-list > dt { + font-weight: bold; + word-break: break-word; + padding-left: 0.5em; + padding-right: 5px; +} + +dl.field-list > dd { + padding-left: 0.5em; + margin-top: 0em; + margin-left: 0em; + margin-bottom: 0em; +} + +dl { + margin-bottom: 15px; +} + +dd > :first-child { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +.sig dd { + margin-top: 0px; + margin-bottom: 0px; +} + +.sig dl { + margin-top: 0px; + margin-bottom: 0px; +} + +dl > dd:last-child, +dl > dd:last-child > :last-child { + margin-bottom: 0; +} + +dt:target, span.highlighted { + background-color: #fbe54e; +} + +rect.highlighted { + fill: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa; +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +.guilabel, .menuselection { + font-family: sans-serif; +} + +.accelerator { + text-decoration: underline; +} + +.classifier { + font-style: oblique; +} + +.classifier:before { + font-style: normal; + margin: 0 0.5em; + content: ":"; + display: inline-block; +} + +abbr, acronym { + border-bottom: dotted 1px; + cursor: help; +} + +.translated { + background-color: rgba(207, 255, 207, 0.2) +} + +.untranslated { + background-color: rgba(255, 207, 207, 0.2) +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; + overflow-y: hidden; /* fixes display issues on Chrome browsers */ +} + +pre, div[class*="highlight-"] { + clear: both; +} + +span.pre { + -moz-hyphens: none; + -ms-hyphens: none; + -webkit-hyphens: none; + hyphens: none; + white-space: nowrap; +} + +div[class*="highlight-"] { + margin: 1em 0; +} + +td.linenos pre { + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + display: block; +} + +table.highlighttable tbody { + display: block; +} + +table.highlighttable tr { + display: flex; +} + +table.highlighttable td { + margin: 0; + padding: 0; +} + +table.highlighttable td.linenos { + padding-right: 0.5em; +} + +table.highlighttable td.code { + flex: 1; + overflow: hidden; +} + +.highlight .hll { + display: block; +} + +div.highlight pre, +table.highlighttable pre { + margin: 0; +} + +div.code-block-caption + div { + margin-top: 0; +} + +div.code-block-caption { + margin-top: 1em; + padding: 2px 5px; + font-size: small; +} + +div.code-block-caption code { + background-color: transparent; +} + +table.highlighttable td.linenos, +span.linenos, +div.highlight span.gp { /* gp: Generic.Prompt */ + user-select: none; + -webkit-user-select: text; /* Safari fallback only */ + -webkit-user-select: none; /* Chrome/Safari */ + -moz-user-select: none; /* Firefox */ + -ms-user-select: none; /* IE10+ */ +} + +div.code-block-caption span.caption-number { + padding: 0.1em 0.3em; + font-style: italic; +} + +div.code-block-caption span.caption-text { +} + +div.literal-block-wrapper { + margin: 1em 0; +} + +code.xref, a code { + background-color: transparent; + font-weight: bold; +} + +h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { + background-color: transparent; +} + +.viewcode-link { + float: right; +} + +.viewcode-back { + float: right; + font-family: sans-serif; +} + +div.viewcode-block:target { + margin: -1px -10px; + padding: 0 10px; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +span.eqno a.headerlink { + position: absolute; + z-index: 1; +} + +div.math:hover a.headerlink { + visibility: visible; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} \ No newline at end of file diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..2a924f1 --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1 @@ +/* This file intentionally left blank. */ diff --git a/docs/_static/doctools.js b/docs/_static/doctools.js new file mode 100644 index 0000000..0398ebb --- /dev/null +++ b/docs/_static/doctools.js @@ -0,0 +1,149 @@ +/* + * Base JavaScript utilities for all Sphinx HTML documentation. + */ +"use strict"; + +const BLACKLISTED_KEY_CONTROL_ELEMENTS = new Set([ + "TEXTAREA", + "INPUT", + "SELECT", + "BUTTON", +]); + +const _ready = (callback) => { + if (document.readyState !== "loading") { + callback(); + } else { + document.addEventListener("DOMContentLoaded", callback); + } +}; + +/** + * Small JavaScript module for the documentation. + */ +const Documentation = { + init: () => { + Documentation.initDomainIndexTable(); + Documentation.initOnKeyListeners(); + }, + + /** + * i18n support + */ + TRANSLATIONS: {}, + PLURAL_EXPR: (n) => (n === 1 ? 0 : 1), + LOCALE: "unknown", + + // gettext and ngettext don't access this so that the functions + // can safely bound to a different name (_ = Documentation.gettext) + gettext: (string) => { + const translated = Documentation.TRANSLATIONS[string]; + switch (typeof translated) { + case "undefined": + return string; // no translation + case "string": + return translated; // translation exists + default: + return translated[0]; // (singular, plural) translation tuple exists + } + }, + + ngettext: (singular, plural, n) => { + const translated = Documentation.TRANSLATIONS[singular]; + if (typeof translated !== "undefined") + return translated[Documentation.PLURAL_EXPR(n)]; + return n === 1 ? singular : plural; + }, + + addTranslations: (catalog) => { + Object.assign(Documentation.TRANSLATIONS, catalog.messages); + Documentation.PLURAL_EXPR = new Function( + "n", + `return (${catalog.plural_expr})` + ); + Documentation.LOCALE = catalog.locale; + }, + + /** + * helper function to focus on search bar + */ + focusSearchBar: () => { + document.querySelectorAll("input[name=q]")[0]?.focus(); + }, + + /** + * Initialise the domain index toggle buttons + */ + initDomainIndexTable: () => { + const toggler = (el) => { + const idNumber = el.id.substr(7); + const toggledRows = document.querySelectorAll(`tr.cg-${idNumber}`); + if (el.src.substr(-9) === "minus.png") { + el.src = `${el.src.substr(0, el.src.length - 9)}plus.png`; + toggledRows.forEach((el) => (el.style.display = "none")); + } else { + el.src = `${el.src.substr(0, el.src.length - 8)}minus.png`; + toggledRows.forEach((el) => (el.style.display = "")); + } + }; + + const togglerElements = document.querySelectorAll("img.toggler"); + togglerElements.forEach((el) => + el.addEventListener("click", (event) => toggler(event.currentTarget)) + ); + togglerElements.forEach((el) => (el.style.display = "")); + if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) togglerElements.forEach(toggler); + }, + + initOnKeyListeners: () => { + // only install a listener if it is really needed + if ( + !DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS && + !DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS + ) + return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; + // bail with special keys + if (event.altKey || event.ctrlKey || event.metaKey) return; + + if (!event.shiftKey) { + switch (event.key) { + case "ArrowLeft": + if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; + + const prevLink = document.querySelector('link[rel="prev"]'); + if (prevLink && prevLink.href) { + window.location.href = prevLink.href; + event.preventDefault(); + } + break; + case "ArrowRight": + if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; + + const nextLink = document.querySelector('link[rel="next"]'); + if (nextLink && nextLink.href) { + window.location.href = nextLink.href; + event.preventDefault(); + } + break; + } + } + + // some keyboard layouts may need Shift to get / + switch (event.key) { + case "/": + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) break; + Documentation.focusSearchBar(); + event.preventDefault(); + } + }); + }, +}; + +// quick alias for translations +const _ = Documentation.gettext; + +_ready(Documentation.init); diff --git a/docs/_static/documentation_options.js b/docs/_static/documentation_options.js new file mode 100644 index 0000000..7e4c114 --- /dev/null +++ b/docs/_static/documentation_options.js @@ -0,0 +1,13 @@ +const DOCUMENTATION_OPTIONS = { + VERSION: '', + LANGUAGE: 'en', + COLLAPSE_INDEX: false, + BUILDER: 'html', + FILE_SUFFIX: '.html', + LINK_SUFFIX: '.html', + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt', + NAVIGATION_WITH_KEYS: false, + SHOW_SEARCH_SUMMARY: true, + ENABLE_SEARCH_SHORTCUTS: true, +}; \ No newline at end of file diff --git a/docs/_static/file.png b/docs/_static/file.png new file mode 100644 index 0000000..a858a41 Binary files /dev/null and b/docs/_static/file.png differ diff --git a/docs/_static/github-banner.svg b/docs/_static/github-banner.svg new file mode 100644 index 0000000..c47d9dc --- /dev/null +++ b/docs/_static/github-banner.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/docs/_static/language_data.js b/docs/_static/language_data.js new file mode 100644 index 0000000..c7fe6c6 --- /dev/null +++ b/docs/_static/language_data.js @@ -0,0 +1,192 @@ +/* + * This script contains the language-specific data used by searchtools.js, + * namely the list of stopwords, stemmer, scorer and splitter. + */ + +var stopwords = ["a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "near", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"]; + + +/* Non-minified version is copied as a separate JS file, if available */ + +/** + * Porter Stemmer + */ +var Stemmer = function() { + + var step2list = { + ational: 'ate', + tional: 'tion', + enci: 'ence', + anci: 'ance', + izer: 'ize', + bli: 'ble', + alli: 'al', + entli: 'ent', + eli: 'e', + ousli: 'ous', + ization: 'ize', + ation: 'ate', + ator: 'ate', + alism: 'al', + iveness: 'ive', + fulness: 'ful', + ousness: 'ous', + aliti: 'al', + iviti: 'ive', + biliti: 'ble', + logi: 'log' + }; + + var step3list = { + icate: 'ic', + ative: '', + alize: 'al', + iciti: 'ic', + ical: 'ic', + ful: '', + ness: '' + }; + + var c = "[^aeiou]"; // consonant + var v = "[aeiouy]"; // vowel + var C = c + "[^aeiouy]*"; // consonant sequence + var V = v + "[aeiou]*"; // vowel sequence + + var mgr0 = "^(" + C + ")?" + V + C; // [C]VC... is m>0 + var meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$"; // [C]VC[V] is m=1 + var mgr1 = "^(" + C + ")?" + V + C + V + C; // [C]VCVC... is m>1 + var s_v = "^(" + C + ")?" + v; // vowel in stem + + this.stemWord = function (w) { + var stem; + var suffix; + var firstch; + var origword = w; + + if (w.length < 3) + return w; + + var re; + var re2; + var re3; + var re4; + + firstch = w.substr(0,1); + if (firstch == "y") + w = firstch.toUpperCase() + w.substr(1); + + // Step 1a + re = /^(.+?)(ss|i)es$/; + re2 = /^(.+?)([^s])s$/; + + if (re.test(w)) + w = w.replace(re,"$1$2"); + else if (re2.test(w)) + w = w.replace(re2,"$1$2"); + + // Step 1b + re = /^(.+?)eed$/; + re2 = /^(.+?)(ed|ing)$/; + if (re.test(w)) { + var fp = re.exec(w); + re = new RegExp(mgr0); + if (re.test(fp[1])) { + re = /.$/; + w = w.replace(re,""); + } + } + else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1]; + re2 = new RegExp(s_v); + if (re2.test(stem)) { + w = stem; + re2 = /(at|bl|iz)$/; + re3 = new RegExp("([^aeiouylsz])\\1$"); + re4 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + if (re2.test(w)) + w = w + "e"; + else if (re3.test(w)) { + re = /.$/; + w = w.replace(re,""); + } + else if (re4.test(w)) + w = w + "e"; + } + } + + // Step 1c + re = /^(.+?)y$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(s_v); + if (re.test(stem)) + w = stem + "i"; + } + + // Step 2 + re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = new RegExp(mgr0); + if (re.test(stem)) + w = stem + step2list[suffix]; + } + + // Step 3 + re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = new RegExp(mgr0); + if (re.test(stem)) + w = stem + step3list[suffix]; + } + + // Step 4 + re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; + re2 = /^(.+?)(s|t)(ion)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(mgr1); + if (re.test(stem)) + w = stem; + } + else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1] + fp[2]; + re2 = new RegExp(mgr1); + if (re2.test(stem)) + w = stem; + } + + // Step 5 + re = /^(.+?)e$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(mgr1); + re2 = new RegExp(meq1); + re3 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) + w = stem; + } + re = /ll$/; + re2 = new RegExp(mgr1); + if (re.test(w) && re2.test(w)) { + re = /.$/; + w = w.replace(re,""); + } + + // and turn initial Y back to y + if (firstch == "y") + w = firstch.toLowerCase() + w.substr(1); + return w; + } +} + diff --git a/docs/_static/minus.png b/docs/_static/minus.png new file mode 100644 index 0000000..d96755f Binary files /dev/null and b/docs/_static/minus.png differ diff --git a/docs/_static/plus.png b/docs/_static/plus.png new file mode 100644 index 0000000..7107cec Binary files /dev/null and b/docs/_static/plus.png differ diff --git a/docs/_static/pygments.css b/docs/_static/pygments.css new file mode 100644 index 0000000..9392ddc --- /dev/null +++ b/docs/_static/pygments.css @@ -0,0 +1,84 @@ +pre { line-height: 125%; } +td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } +span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } +td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } +span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } +.highlight .hll { background-color: #ffffcc } +.highlight { background: #f8f8f8; } +.highlight .c { color: #8F5902; font-style: italic } /* Comment */ +.highlight .err { color: #A40000; border: 1px solid #EF2929 } /* Error */ +.highlight .g { color: #000 } /* Generic */ +.highlight .k { color: #004461; font-weight: bold } /* Keyword */ +.highlight .l { color: #000 } /* Literal */ +.highlight .n { color: #000 } /* Name */ +.highlight .o { color: #582800 } /* Operator */ +.highlight .x { color: #000 } /* Other */ +.highlight .p { color: #000; font-weight: bold } /* Punctuation */ +.highlight .ch { color: #8F5902; font-style: italic } /* Comment.Hashbang */ +.highlight .cm { color: #8F5902; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #8F5902 } /* Comment.Preproc */ +.highlight .cpf { color: #8F5902; font-style: italic } /* Comment.PreprocFile */ +.highlight .c1 { color: #8F5902; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #8F5902; font-style: italic } /* Comment.Special */ +.highlight .gd { color: #A40000 } /* Generic.Deleted */ +.highlight .ge { color: #000; font-style: italic } /* Generic.Emph */ +.highlight .ges { color: #000 } /* Generic.EmphStrong */ +.highlight .gr { color: #EF2929 } /* Generic.Error */ +.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.highlight .gi { color: #00A000 } /* Generic.Inserted */ +.highlight .go { color: #888 } /* Generic.Output */ +.highlight .gp { color: #745334 } /* Generic.Prompt */ +.highlight .gs { color: #000; font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.highlight .gt { color: #A40000; font-weight: bold } /* Generic.Traceback */ +.highlight .kc { color: #004461; font-weight: bold } /* Keyword.Constant */ +.highlight .kd { color: #004461; font-weight: bold } /* Keyword.Declaration */ +.highlight .kn { color: #004461; font-weight: bold } /* Keyword.Namespace */ +.highlight .kp { color: #004461; font-weight: bold } /* Keyword.Pseudo */ +.highlight .kr { color: #004461; font-weight: bold } /* Keyword.Reserved */ +.highlight .kt { color: #004461; font-weight: bold } /* Keyword.Type */ +.highlight .ld { color: #000 } /* Literal.Date */ +.highlight .m { color: #900 } /* Literal.Number */ +.highlight .s { color: #4E9A06 } /* Literal.String */ +.highlight .na { color: #C4A000 } /* Name.Attribute */ +.highlight .nb { color: #004461 } /* Name.Builtin */ +.highlight .nc { color: #000 } /* Name.Class */ +.highlight .no { color: #000 } /* Name.Constant */ +.highlight .nd { color: #888 } /* Name.Decorator */ +.highlight .ni { color: #CE5C00 } /* Name.Entity */ +.highlight .ne { color: #C00; font-weight: bold } /* Name.Exception */ +.highlight .nf { color: #000 } /* Name.Function */ +.highlight .nl { color: #F57900 } /* Name.Label */ +.highlight .nn { color: #000 } /* Name.Namespace */ +.highlight .nx { color: #000 } /* Name.Other */ +.highlight .py { color: #000 } /* Name.Property */ +.highlight .nt { color: #004461; font-weight: bold } /* Name.Tag */ +.highlight .nv { color: #000 } /* Name.Variable */ +.highlight .ow { color: #004461; font-weight: bold } /* Operator.Word */ +.highlight .pm { color: #000; font-weight: bold } /* Punctuation.Marker */ +.highlight .w { color: #F8F8F8 } /* Text.Whitespace */ +.highlight .mb { color: #900 } /* Literal.Number.Bin */ +.highlight .mf { color: #900 } /* Literal.Number.Float */ +.highlight .mh { color: #900 } /* Literal.Number.Hex */ +.highlight .mi { color: #900 } /* Literal.Number.Integer */ +.highlight .mo { color: #900 } /* Literal.Number.Oct */ +.highlight .sa { color: #4E9A06 } /* Literal.String.Affix */ +.highlight .sb { color: #4E9A06 } /* Literal.String.Backtick */ +.highlight .sc { color: #4E9A06 } /* Literal.String.Char */ +.highlight .dl { color: #4E9A06 } /* Literal.String.Delimiter */ +.highlight .sd { color: #8F5902; font-style: italic } /* Literal.String.Doc */ +.highlight .s2 { color: #4E9A06 } /* Literal.String.Double */ +.highlight .se { color: #4E9A06 } /* Literal.String.Escape */ +.highlight .sh { color: #4E9A06 } /* Literal.String.Heredoc */ +.highlight .si { color: #4E9A06 } /* Literal.String.Interpol */ +.highlight .sx { color: #4E9A06 } /* Literal.String.Other */ +.highlight .sr { color: #4E9A06 } /* Literal.String.Regex */ +.highlight .s1 { color: #4E9A06 } /* Literal.String.Single */ +.highlight .ss { color: #4E9A06 } /* Literal.String.Symbol */ +.highlight .bp { color: #3465A4 } /* Name.Builtin.Pseudo */ +.highlight .fm { color: #000 } /* Name.Function.Magic */ +.highlight .vc { color: #000 } /* Name.Variable.Class */ +.highlight .vg { color: #000 } /* Name.Variable.Global */ +.highlight .vi { color: #000 } /* Name.Variable.Instance */ +.highlight .vm { color: #000 } /* Name.Variable.Magic */ +.highlight .il { color: #900 } /* Literal.Number.Integer.Long */ \ No newline at end of file diff --git a/docs/_static/searchtools.js b/docs/_static/searchtools.js new file mode 100644 index 0000000..2c774d1 --- /dev/null +++ b/docs/_static/searchtools.js @@ -0,0 +1,632 @@ +/* + * Sphinx JavaScript utilities for the full-text search. + */ +"use strict"; + +/** + * Simple result scoring code. + */ +if (typeof Scorer === "undefined") { + var Scorer = { + // Implement the following function to further tweak the score for each result + // The function takes a result array [docname, title, anchor, descr, score, filename] + // and returns the new score. + /* + score: result => { + const [docname, title, anchor, descr, score, filename, kind] = result + return score + }, + */ + + // query matches the full name of an object + objNameMatch: 11, + // or matches in the last dotted part of the object name + objPartialMatch: 6, + // Additive scores depending on the priority of the object + objPrio: { + 0: 15, // used to be importantResults + 1: 5, // used to be objectResults + 2: -5, // used to be unimportantResults + }, + // Used when the priority is not in the mapping. + objPrioDefault: 0, + + // query found in title + title: 15, + partialTitle: 7, + // query found in terms + term: 5, + partialTerm: 2, + }; +} + +// Global search result kind enum, used by themes to style search results. +class SearchResultKind { + static get index() { return "index"; } + static get object() { return "object"; } + static get text() { return "text"; } + static get title() { return "title"; } +} + +const _removeChildren = (element) => { + while (element && element.lastChild) element.removeChild(element.lastChild); +}; + +/** + * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + */ +const _escapeRegExp = (string) => + string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string + +const _displayItem = (item, searchTerms, highlightTerms) => { + const docBuilder = DOCUMENTATION_OPTIONS.BUILDER; + const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX; + const docLinkSuffix = DOCUMENTATION_OPTIONS.LINK_SUFFIX; + const showSearchSummary = DOCUMENTATION_OPTIONS.SHOW_SEARCH_SUMMARY; + const contentRoot = document.documentElement.dataset.content_root; + + const [docName, title, anchor, descr, score, _filename, kind] = item; + + let listItem = document.createElement("li"); + // Add a class representing the item's type: + // can be used by a theme's CSS selector for styling + // See SearchResultKind for the class names. + listItem.classList.add(`kind-${kind}`); + let requestUrl; + let linkUrl; + if (docBuilder === "dirhtml") { + // dirhtml builder + let dirname = docName + "/"; + if (dirname.match(/\/index\/$/)) + dirname = dirname.substring(0, dirname.length - 6); + else if (dirname === "index/") dirname = ""; + requestUrl = contentRoot + dirname; + linkUrl = requestUrl; + } else { + // normal html builders + requestUrl = contentRoot + docName + docFileSuffix; + linkUrl = docName + docLinkSuffix; + } + let linkEl = listItem.appendChild(document.createElement("a")); + linkEl.href = linkUrl + anchor; + linkEl.dataset.score = score; + linkEl.innerHTML = title; + if (descr) { + listItem.appendChild(document.createElement("span")).innerHTML = + " (" + descr + ")"; + // highlight search terms in the description + if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js + highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted")); + } + else if (showSearchSummary) + fetch(requestUrl) + .then((responseData) => responseData.text()) + .then((data) => { + if (data) + listItem.appendChild( + Search.makeSearchSummary(data, searchTerms, anchor) + ); + // highlight search terms in the summary + if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js + highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted")); + }); + Search.output.appendChild(listItem); +}; +const _finishSearch = (resultCount) => { + Search.stopPulse(); + Search.title.innerText = _("Search Results"); + if (!resultCount) + Search.status.innerText = Documentation.gettext( + "Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories." + ); + else + Search.status.innerText = Documentation.ngettext( + "Search finished, found one page matching the search query.", + "Search finished, found ${resultCount} pages matching the search query.", + resultCount, + ).replace('${resultCount}', resultCount); +}; +const _displayNextItem = ( + results, + resultCount, + searchTerms, + highlightTerms, +) => { + // results left, load the summary and display it + // this is intended to be dynamic (don't sub resultsCount) + if (results.length) { + _displayItem(results.pop(), searchTerms, highlightTerms); + setTimeout( + () => _displayNextItem(results, resultCount, searchTerms, highlightTerms), + 5 + ); + } + // search finished, update title and status message + else _finishSearch(resultCount); +}; +// Helper function used by query() to order search results. +// Each input is an array of [docname, title, anchor, descr, score, filename, kind]. +// Order the results by score (in opposite order of appearance, since the +// `_displayNextItem` function uses pop() to retrieve items) and then alphabetically. +const _orderResultsByScoreThenName = (a, b) => { + const leftScore = a[4]; + const rightScore = b[4]; + if (leftScore === rightScore) { + // same score: sort alphabetically + const leftTitle = a[1].toLowerCase(); + const rightTitle = b[1].toLowerCase(); + if (leftTitle === rightTitle) return 0; + return leftTitle > rightTitle ? -1 : 1; // inverted is intentional + } + return leftScore > rightScore ? 1 : -1; +}; + +/** + * Default splitQuery function. Can be overridden in ``sphinx.search`` with a + * custom function per language. + * + * The regular expression works by splitting the string on consecutive characters + * that are not Unicode letters, numbers, underscores, or emoji characters. + * This is the same as ``\W+`` in Python, preserving the surrogate pair area. + */ +if (typeof splitQuery === "undefined") { + var splitQuery = (query) => query + .split(/[^\p{Letter}\p{Number}_\p{Emoji_Presentation}]+/gu) + .filter(term => term) // remove remaining empty strings +} + +/** + * Search Module + */ +const Search = { + _index: null, + _queued_query: null, + _pulse_status: -1, + + htmlToText: (htmlString, anchor) => { + const htmlElement = new DOMParser().parseFromString(htmlString, 'text/html'); + for (const removalQuery of [".headerlink", "script", "style"]) { + htmlElement.querySelectorAll(removalQuery).forEach((el) => { el.remove() }); + } + if (anchor) { + const anchorContent = htmlElement.querySelector(`[role="main"] ${anchor}`); + if (anchorContent) return anchorContent.textContent; + + console.warn( + `Anchored content block not found. Sphinx search tries to obtain it via DOM query '[role=main] ${anchor}'. Check your theme or template.` + ); + } + + // if anchor not specified or not found, fall back to main content + const docContent = htmlElement.querySelector('[role="main"]'); + if (docContent) return docContent.textContent; + + console.warn( + "Content block not found. Sphinx search tries to obtain it via DOM query '[role=main]'. Check your theme or template." + ); + return ""; + }, + + init: () => { + const query = new URLSearchParams(window.location.search).get("q"); + document + .querySelectorAll('input[name="q"]') + .forEach((el) => (el.value = query)); + if (query) Search.performSearch(query); + }, + + loadIndex: (url) => + (document.body.appendChild(document.createElement("script")).src = url), + + setIndex: (index) => { + Search._index = index; + if (Search._queued_query !== null) { + const query = Search._queued_query; + Search._queued_query = null; + Search.query(query); + } + }, + + hasIndex: () => Search._index !== null, + + deferQuery: (query) => (Search._queued_query = query), + + stopPulse: () => (Search._pulse_status = -1), + + startPulse: () => { + if (Search._pulse_status >= 0) return; + + const pulse = () => { + Search._pulse_status = (Search._pulse_status + 1) % 4; + Search.dots.innerText = ".".repeat(Search._pulse_status); + if (Search._pulse_status >= 0) window.setTimeout(pulse, 500); + }; + pulse(); + }, + + /** + * perform a search for something (or wait until index is loaded) + */ + performSearch: (query) => { + // create the required interface elements + const searchText = document.createElement("h2"); + searchText.textContent = _("Searching"); + const searchSummary = document.createElement("p"); + searchSummary.classList.add("search-summary"); + searchSummary.innerText = ""; + const searchList = document.createElement("ul"); + searchList.setAttribute("role", "list"); + searchList.classList.add("search"); + + const out = document.getElementById("search-results"); + Search.title = out.appendChild(searchText); + Search.dots = Search.title.appendChild(document.createElement("span")); + Search.status = out.appendChild(searchSummary); + Search.output = out.appendChild(searchList); + + const searchProgress = document.getElementById("search-progress"); + // Some themes don't use the search progress node + if (searchProgress) { + searchProgress.innerText = _("Preparing search..."); + } + Search.startPulse(); + + // index already loaded, the browser was quick! + if (Search.hasIndex()) Search.query(query); + else Search.deferQuery(query); + }, + + _parseQuery: (query) => { + // stem the search terms and add them to the correct list + const stemmer = new Stemmer(); + const searchTerms = new Set(); + const excludedTerms = new Set(); + const highlightTerms = new Set(); + const objectTerms = new Set(splitQuery(query.toLowerCase().trim())); + splitQuery(query.trim()).forEach((queryTerm) => { + const queryTermLower = queryTerm.toLowerCase(); + + // maybe skip this "word" + // stopwords array is from language_data.js + if ( + stopwords.indexOf(queryTermLower) !== -1 || + queryTerm.match(/^\d+$/) + ) + return; + + // stem the word + let word = stemmer.stemWord(queryTermLower); + // select the correct list + if (word[0] === "-") excludedTerms.add(word.substr(1)); + else { + searchTerms.add(word); + highlightTerms.add(queryTermLower); + } + }); + + if (SPHINX_HIGHLIGHT_ENABLED) { // set in sphinx_highlight.js + localStorage.setItem("sphinx_highlight_terms", [...highlightTerms].join(" ")) + } + + // console.debug("SEARCH: searching for:"); + // console.info("required: ", [...searchTerms]); + // console.info("excluded: ", [...excludedTerms]); + + return [query, searchTerms, excludedTerms, highlightTerms, objectTerms]; + }, + + /** + * execute search (requires search index to be loaded) + */ + _performSearch: (query, searchTerms, excludedTerms, highlightTerms, objectTerms) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + const allTitles = Search._index.alltitles; + const indexEntries = Search._index.indexentries; + + // Collect multiple result groups to be sorted separately and then ordered. + // Each is an array of [docname, title, anchor, descr, score, filename, kind]. + const normalResults = []; + const nonMainIndexResults = []; + + _removeChildren(document.getElementById("search-progress")); + + const queryLower = query.toLowerCase().trim(); + for (const [title, foundTitles] of Object.entries(allTitles)) { + if (title.toLowerCase().trim().includes(queryLower) && (queryLower.length >= title.length/2)) { + for (const [file, id] of foundTitles) { + const score = Math.round(Scorer.title * queryLower.length / title.length); + const boost = titles[file] === title ? 1 : 0; // add a boost for document titles + normalResults.push([ + docNames[file], + titles[file] !== title ? `${titles[file]} > ${title}` : title, + id !== null ? "#" + id : "", + null, + score + boost, + filenames[file], + SearchResultKind.title, + ]); + } + } + } + + // search for explicit entries in index directives + for (const [entry, foundEntries] of Object.entries(indexEntries)) { + if (entry.includes(queryLower) && (queryLower.length >= entry.length/2)) { + for (const [file, id, isMain] of foundEntries) { + const score = Math.round(100 * queryLower.length / entry.length); + const result = [ + docNames[file], + titles[file], + id ? "#" + id : "", + null, + score, + filenames[file], + SearchResultKind.index, + ]; + if (isMain) { + normalResults.push(result); + } else { + nonMainIndexResults.push(result); + } + } + } + } + + // lookup as object + objectTerms.forEach((term) => + normalResults.push(...Search.performObjectSearch(term, objectTerms)) + ); + + // lookup as search terms in fulltext + normalResults.push(...Search.performTermsSearch(searchTerms, excludedTerms)); + + // let the scorer override scores with a custom scoring function + if (Scorer.score) { + normalResults.forEach((item) => (item[4] = Scorer.score(item))); + nonMainIndexResults.forEach((item) => (item[4] = Scorer.score(item))); + } + + // Sort each group of results by score and then alphabetically by name. + normalResults.sort(_orderResultsByScoreThenName); + nonMainIndexResults.sort(_orderResultsByScoreThenName); + + // Combine the result groups in (reverse) order. + // Non-main index entries are typically arbitrary cross-references, + // so display them after other results. + let results = [...nonMainIndexResults, ...normalResults]; + + // remove duplicate search results + // note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept + let seen = new Set(); + results = results.reverse().reduce((acc, result) => { + let resultStr = result.slice(0, 4).concat([result[5]]).map(v => String(v)).join(','); + if (!seen.has(resultStr)) { + acc.push(result); + seen.add(resultStr); + } + return acc; + }, []); + + return results.reverse(); + }, + + query: (query) => { + const [searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms] = Search._parseQuery(query); + const results = Search._performSearch(searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms); + + // for debugging + //Search.lastresults = results.slice(); // a copy + // console.info("search results:", Search.lastresults); + + // print the results + _displayNextItem(results, results.length, searchTerms, highlightTerms); + }, + + /** + * search for object names + */ + performObjectSearch: (object, objectTerms) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const objects = Search._index.objects; + const objNames = Search._index.objnames; + const titles = Search._index.titles; + + const results = []; + + const objectSearchCallback = (prefix, match) => { + const name = match[4] + const fullname = (prefix ? prefix + "." : "") + name; + const fullnameLower = fullname.toLowerCase(); + if (fullnameLower.indexOf(object) < 0) return; + + let score = 0; + const parts = fullnameLower.split("."); + + // check for different match types: exact matches of full name or + // "last name" (i.e. last dotted part) + if (fullnameLower === object || parts.slice(-1)[0] === object) + score += Scorer.objNameMatch; + else if (parts.slice(-1)[0].indexOf(object) > -1) + score += Scorer.objPartialMatch; // matches in last name + + const objName = objNames[match[1]][2]; + const title = titles[match[0]]; + + // If more than one term searched for, we require other words to be + // found in the name/title/description + const otherTerms = new Set(objectTerms); + otherTerms.delete(object); + if (otherTerms.size > 0) { + const haystack = `${prefix} ${name} ${objName} ${title}`.toLowerCase(); + if ( + [...otherTerms].some((otherTerm) => haystack.indexOf(otherTerm) < 0) + ) + return; + } + + let anchor = match[3]; + if (anchor === "") anchor = fullname; + else if (anchor === "-") anchor = objNames[match[1]][1] + "-" + fullname; + + const descr = objName + _(", in ") + title; + + // add custom score for some objects according to scorer + if (Scorer.objPrio.hasOwnProperty(match[2])) + score += Scorer.objPrio[match[2]]; + else score += Scorer.objPrioDefault; + + results.push([ + docNames[match[0]], + fullname, + "#" + anchor, + descr, + score, + filenames[match[0]], + SearchResultKind.object, + ]); + }; + Object.keys(objects).forEach((prefix) => + objects[prefix].forEach((array) => + objectSearchCallback(prefix, array) + ) + ); + return results; + }, + + /** + * search for full-text terms in the index + */ + performTermsSearch: (searchTerms, excludedTerms) => { + // prepare search + const terms = Search._index.terms; + const titleTerms = Search._index.titleterms; + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + + const scoreMap = new Map(); + const fileMap = new Map(); + + // perform the search on the required terms + searchTerms.forEach((word) => { + const files = []; + const arr = [ + { files: terms[word], score: Scorer.term }, + { files: titleTerms[word], score: Scorer.title }, + ]; + // add support for partial matches + if (word.length > 2) { + const escapedWord = _escapeRegExp(word); + if (!terms.hasOwnProperty(word)) { + Object.keys(terms).forEach((term) => { + if (term.match(escapedWord)) + arr.push({ files: terms[term], score: Scorer.partialTerm }); + }); + } + if (!titleTerms.hasOwnProperty(word)) { + Object.keys(titleTerms).forEach((term) => { + if (term.match(escapedWord)) + arr.push({ files: titleTerms[term], score: Scorer.partialTitle }); + }); + } + } + + // no match but word was a required one + if (arr.every((record) => record.files === undefined)) return; + + // found search word in contents + arr.forEach((record) => { + if (record.files === undefined) return; + + let recordFiles = record.files; + if (recordFiles.length === undefined) recordFiles = [recordFiles]; + files.push(...recordFiles); + + // set score for the word in each file + recordFiles.forEach((file) => { + if (!scoreMap.has(file)) scoreMap.set(file, {}); + scoreMap.get(file)[word] = record.score; + }); + }); + + // create the mapping + files.forEach((file) => { + if (!fileMap.has(file)) fileMap.set(file, [word]); + else if (fileMap.get(file).indexOf(word) === -1) fileMap.get(file).push(word); + }); + }); + + // now check if the files don't contain excluded terms + const results = []; + for (const [file, wordList] of fileMap) { + // check if all requirements are matched + + // as search terms with length < 3 are discarded + const filteredTermCount = [...searchTerms].filter( + (term) => term.length > 2 + ).length; + if ( + wordList.length !== searchTerms.size && + wordList.length !== filteredTermCount + ) + continue; + + // ensure that none of the excluded terms is in the search result + if ( + [...excludedTerms].some( + (term) => + terms[term] === file || + titleTerms[term] === file || + (terms[term] || []).includes(file) || + (titleTerms[term] || []).includes(file) + ) + ) + break; + + // select one (max) score for the file. + const score = Math.max(...wordList.map((w) => scoreMap.get(file)[w])); + // add result to the result list + results.push([ + docNames[file], + titles[file], + "", + null, + score, + filenames[file], + SearchResultKind.text, + ]); + } + return results; + }, + + /** + * helper function to return a node containing the + * search summary for a given text. keywords is a list + * of stemmed words. + */ + makeSearchSummary: (htmlText, keywords, anchor) => { + const text = Search.htmlToText(htmlText, anchor); + if (text === "") return null; + + const textLower = text.toLowerCase(); + const actualStartPosition = [...keywords] + .map((k) => textLower.indexOf(k.toLowerCase())) + .filter((i) => i > -1) + .slice(-1)[0]; + const startWithContext = Math.max(actualStartPosition - 120, 0); + + const top = startWithContext === 0 ? "" : "..."; + const tail = startWithContext + 240 < text.length ? "..." : ""; + + let summary = document.createElement("p"); + summary.classList.add("context"); + summary.textContent = top + text.substr(startWithContext, 240).trim() + tail; + + return summary; + }, +}; + +_ready(Search.init); diff --git a/docs/_static/sphinx_highlight.js b/docs/_static/sphinx_highlight.js new file mode 100644 index 0000000..8a96c69 --- /dev/null +++ b/docs/_static/sphinx_highlight.js @@ -0,0 +1,154 @@ +/* Highlighting utilities for Sphinx HTML documentation. */ +"use strict"; + +const SPHINX_HIGHLIGHT_ENABLED = true + +/** + * highlight a given string on a node by wrapping it in + * span elements with the given class name. + */ +const _highlight = (node, addItems, text, className) => { + if (node.nodeType === Node.TEXT_NODE) { + const val = node.nodeValue; + const parent = node.parentNode; + const pos = val.toLowerCase().indexOf(text); + if ( + pos >= 0 && + !parent.classList.contains(className) && + !parent.classList.contains("nohighlight") + ) { + let span; + + const closestNode = parent.closest("body, svg, foreignObject"); + const isInSVG = closestNode && closestNode.matches("svg"); + if (isInSVG) { + span = document.createElementNS("http://www.w3.org/2000/svg", "tspan"); + } else { + span = document.createElement("span"); + span.classList.add(className); + } + + span.appendChild(document.createTextNode(val.substr(pos, text.length))); + const rest = document.createTextNode(val.substr(pos + text.length)); + parent.insertBefore( + span, + parent.insertBefore( + rest, + node.nextSibling + ) + ); + node.nodeValue = val.substr(0, pos); + /* There may be more occurrences of search term in this node. So call this + * function recursively on the remaining fragment. + */ + _highlight(rest, addItems, text, className); + + if (isInSVG) { + const rect = document.createElementNS( + "http://www.w3.org/2000/svg", + "rect" + ); + const bbox = parent.getBBox(); + rect.x.baseVal.value = bbox.x; + rect.y.baseVal.value = bbox.y; + rect.width.baseVal.value = bbox.width; + rect.height.baseVal.value = bbox.height; + rect.setAttribute("class", className); + addItems.push({ parent: parent, target: rect }); + } + } + } else if (node.matches && !node.matches("button, select, textarea")) { + node.childNodes.forEach((el) => _highlight(el, addItems, text, className)); + } +}; +const _highlightText = (thisNode, text, className) => { + let addItems = []; + _highlight(thisNode, addItems, text, className); + addItems.forEach((obj) => + obj.parent.insertAdjacentElement("beforebegin", obj.target) + ); +}; + +/** + * Small JavaScript module for the documentation. + */ +const SphinxHighlight = { + + /** + * highlight the search words provided in localstorage in the text + */ + highlightSearchWords: () => { + if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight + + // get and clear terms from localstorage + const url = new URL(window.location); + const highlight = + localStorage.getItem("sphinx_highlight_terms") + || url.searchParams.get("highlight") + || ""; + localStorage.removeItem("sphinx_highlight_terms") + url.searchParams.delete("highlight"); + window.history.replaceState({}, "", url); + + // get individual terms from highlight string + const terms = highlight.toLowerCase().split(/\s+/).filter(x => x); + if (terms.length === 0) return; // nothing to do + + // There should never be more than one element matching "div.body" + const divBody = document.querySelectorAll("div.body"); + const body = divBody.length ? divBody[0] : document.querySelector("body"); + window.setTimeout(() => { + terms.forEach((term) => _highlightText(body, term, "highlighted")); + }, 10); + + const searchBox = document.getElementById("searchbox"); + if (searchBox === null) return; + searchBox.appendChild( + document + .createRange() + .createContextualFragment( + '

' + + '' + + _("Hide Search Matches") + + "

" + ) + ); + }, + + /** + * helper function to hide the search marks again + */ + hideSearchWords: () => { + document + .querySelectorAll("#searchbox .highlight-link") + .forEach((el) => el.remove()); + document + .querySelectorAll("span.highlighted") + .forEach((el) => el.classList.remove("highlighted")); + localStorage.removeItem("sphinx_highlight_terms") + }, + + initEscapeListener: () => { + // only install a listener if it is really needed + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; + // bail with special keys + if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return; + if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) { + SphinxHighlight.hideSearchWords(); + event.preventDefault(); + } + }); + }, +}; + +_ready(() => { + /* Do not call highlightSearchWords() when we are on the search page. + * It will highlight words from the *previous* search query. + */ + if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords(); + SphinxHighlight.initEscapeListener(); +}); diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 6beb2e9..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,73 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -from typing import List - -sys.path.insert(0, os.path.abspath("../")) - -# -- Project information ----------------------------------------------------- - -project = "fedcloudclient" -copyright = "2022, Viet Tran" -author = "Viet Tran" - -# The full version, including alpha/beta/rc tags -# release = '0.0.2-dev15' - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.todo", - "sphinx.ext.viewcode", - "sphinx.ext.autodoc", -] - -smartquotes = False - -source_suffix = [".rst", ".md"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns: List[str] = [] - -# -- Options for HTML output ------------------------------------------------- - -html_context = { - "display_github": True, - "github_user": "tdviet", - "github_repo": "fedcloudclient", - "github_version": "master/docs/", -} - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "sphinx_rtd_theme" -html_logo = "fedcloudclient-logo-non-transparent-small.png" -html_theme_options = { - "logo_only": True, - "display_version": False, -} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] diff --git a/docs/docs/FAQ.html b/docs/docs/FAQ.html new file mode 100644 index 0000000..44d7969 --- /dev/null +++ b/docs/docs/FAQ.html @@ -0,0 +1,120 @@ + + + + + + + + FAQ and Troubleshooting — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

FAQ and Troubleshooting

+
    +

  1. FedCloud client gives error message “SSL exception connecting to https:// …” when connecting to some sites

    2. +
+

Some sites use certificates issued by national certificate authorities that are not included in the default +OS distribution. If you receive error message “SSL exception connecting to https:// …”, +follow instructions +for installing EGI Core Trust Anchor certificates and add them to the certificate bundle of Python requests. For quick +test in virtual environment, just execute the following commands. See this +README.md for more details.

+
$ wget https://raw.githubusercontent.com/tdviet/python-requests-bundle-certs/main/scripts/install_certs.sh
+$ bash install_certs.sh
+
+
+
    +

  1. FedCloud client frozen during initialization (mainly on virtual machines at some sites)

    2. +
+

It is a known problem of libsodium which is used by oidc-agent Python library. The problem is described +here. Check the entropy on the VMs by executing command +“cat /proc/sys/kernel/random/entropy_avail”, and if the result is lower than 300, install haveged or rng-tools. +On VMs with Centos, you also have to start the daemon manually after installation (or reboot the VMs)

+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/cheat.html b/docs/docs/cheat.html new file mode 100644 index 0000000..1b0fc6c --- /dev/null +++ b/docs/docs/cheat.html @@ -0,0 +1,349 @@ + + + + + + + + Cheat sheet — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Cheat sheet

+

See Tutorial +for more details of commands.

+
+

Local install via pip3

+
    +

  • Create a Python virtual environment:

    • +
+
$ python3 -m venv env
+
+
+
    +

  • Activate the virtual environment

    • +
+
$ source env/bin/activate
+
+
+
    +

  • Install fedcloudclient via pip:

    • +
+
$ pip3 install fedcloudclient
+
+
+
    +

  • Install IGTF certificates:

    • +
+
$ wget https://raw.githubusercontent.com/tdviet/python-requests-bundle-certs/main/scripts/install_certs.sh
+$ bash install_certs.sh
+
+
+
+
+

Using Docker container

+
    +

  • Pull the latest version of fedcloudclient container

    • +
+
$ sudo docker pull tdviet/fedcloudclient
+
+
+
    +

  • Start fedcloudclient container with oidc-agent account:

    • +
+
$ sudo docker run -it -v ~/.config/oidc-agent/egi:/root/.config/oidc-agent/egi --name fedcloud tdviet/fedcloudclient bash
+
+
+
    +

  • Restart previously terminated container:

    • +
+
$ sudo docker start -i fedcloud
+
+
+
+
+

Using oidc-agent

+
    +

  • Create an oidc-agent account (if not done):

    • +
+
$ oidc-gen --pub --issuer https://aai.egi.eu/auth/realms/egi --scope "eduperson_entitlement email" egi
+
+
+
    +

  • Load oidc-agent account and set environment for fedcloudclient:

    • +
+
$ eval `oidc-keychain --accounts egi` && export OIDC_AGENT_ACCOUNT=egi
+
+
+
+
+

Basic usages

+
    +

  • List your VO memberships according to the access token:

    • +
+
$ fedcloud token list-vos
+
+
+
    +

  • List sites in the EGI Federated Cloud:

    • +
+
$ fedcloud site list
+
+
+
    +

  • List all sites supporting a Virtual Organization in the EGI Federated Cloud:

    • +
+
$ fedcloud site list --vo vo.access.egi.eu
+
+
+
    +

  • Execute an OpenStack command:

    • +
+
$ fedcloud openstack image list --site IISAS-FedCloud --vo eosc-synergy.eu
+
+
+
    +

  • Execute an OpenStack command on all sites:

    • +
+
$ fedcloud openstack server list --site ALL_SITES --vo eosc-synergy.eu
+
+
+
    +

  • Print only selected values (for scripting):

    • +
+
$ export OS_TOKEN=$(fedcloud openstack --site CESGA --vo vo.access.egi.eu token issue -c id -f value)
+
+
+
    +

  • All-sites commands with full JSON output:

    • +
+
$ fedcloud openstack image list --site ALL_SITES --vo eosc-synergy.eu --json-output
+
+
+
+
+

Searching and selecting resources

+
    +

  • Show all available projects:

    • +
+
$ fedcloud endpoint projects --site ALL_SITES
+
+
+
    +

  • Show all Horizon dashboards:

    • +
+
$ fedcloud endpoint list --service-type org.openstack.horizon --site ALL_SITES
+
+
+
    +

  • Search images with appliance title in AppDB:

    • +
+
$ fedcloud openstack image list --property "dc:title"="Image for EGI Docker [Ubuntu/18.04/VirtualBox]" --site CESNET-MCC  --vo eosc-synergy.eu
+
+
+
    +

  • Select flavors with 2 CPUs and RAM >= 2048 on a site/VO:

    • +
+
$ fedcloud select flavor --site IISAS-FedCloud --vo vo.access.egi.eu --vcpus 2 --flavor-specs "RAM>=2048" --output-format list
+
+
+
    +

  • Select EGI Ubuntu 20.04 images on a site/VO:

    • +
+
# Simpler but longer way
+$ fedcloud select image --site IFCA-LCG2 --vo training.egi.eu --image-specs "Name =~ Ubuntu" --image-specs "Name =~ '20.04'" --image-specs "Name =~ EGI" --output-format list
+
+
+
# Shorter but more complex regex
+$ fedcloud select image --site IFCA-LCG2 --vo training.egi.eu --image-specs "Name =~ 'EGI.*Ubuntu.*20.04'"  --output-format list
+
+
+
+
+

Mapping and filtering results from OpenStack commands

+
    +

  • Select flavors with 2 CPUs:

    • +
+
$ fedcloud openstack flavor list  --site IISAS-FedCloud --vo eosc-synergy.eu --json-output | \
+  jq -r  '.[].Result[] | select(.VCPUs == 2) | .Name'
+
+
+
    +

  • Select GPU flavors and show their GPU properties on a site:

    • +
+
$ fedcloud openstack flavor list --long --site IISAS-FedCloud --vo acc-comp.egi.eu --json-output | \
+  jq -r '.[].Result | map(select(.Properties."Accelerator:Type" == "GPU")) | .'
+
+
+
    +

  • Select GPU flavors and show their GPU properties on all sites:

    • +
+
$ fedcloud openstack flavor list --long --site ALL_SITES --vo vo.access.egi.eu --json-output | \
+  jq -r 'map(select(."Error code" ==  0)) |
+         map(.Result = (.Result| map(select(.Properties."Accelerator:Type" == "GPU")))) |
+         map(select(.Result | length >  0))'
+
+
+
    +

  • Construct JSON objects just with site names and flavor names, remove all other properties:

    • +
+
$ fedcloud openstack flavor list --long --site ALL_SITES --vo vo.access.egi.eu --json-output | \
+  jq -r 'map(select(."Error code" ==  0)) |
+         map({Site:.Site, Flavors:[.Result[].Name]})'
+
+
+
+
+

Useful commands

+
    +

  • Check expiration time of access token (not work for oidc-agent-account):

    • +
+
$ fedcloud token check
+
+
+
    +

  • Set OpenStack environment variables:

    • +
+
$ eval $(fedcloud site env --site IISAS-FedCloud --vo vo.access.egi.eu)
+
+
+
    +

  • List all my own VMs:

    • +
+
$  list-all-my-own-vms.sh --vo fedcloud.egi.eu
+
+
+
    +

  • Activate shell completion

    • +
+
# Quick and dirty way (may be resulted in unresponsive shell)
+$ eval "$(_FEDCLOUD_COMPLETE=bash_source fedcloud)"
+
+
+
# More systematic way
+$ wget https://raw.githubusercontent.com/tdviet/fedcloudclient/master/examples/fedcloud_bash_completion.sh
+$ source fedcloud_bash_completion.sh
+
+
+
    +

  • Pass a mytoken to Virtual Machines in the EGI Federated Cloud

    • +
+
# Create the file "user.txt" with
+$ cat user.txt
+FEDCLOUD_MYTOKEN=<mytoken> # created on https://mytoken.data.kit.edu/
+
+# Pass it to OpenStack
+FEDCLOUD_SITE=IISAS-FedCloud
+FEDCLOUD_VO=vo.access.egi.eu
+fedcloud openstack server create --flavor <flavor> --image <image> --user-data user.txt --key-name <keypair> testvm
+
+# Once you log into the VM you can retrieve the "mytoken" with
+curl http://169.254.169.254/openstack/latest/user_data/
+
+# and use it with
+FEDCLOUD_MYTOKEN=<mytoken> # copied from the previous curl command
+fedcloud token check
+
+
+
+
+

More information

+
    +

  • Get help:

    • +
+
$ fedcloud --help
+$ fedcloud site --help
+
+
+ +
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/development.html b/docs/docs/development.html new file mode 100644 index 0000000..451c6e4 --- /dev/null +++ b/docs/docs/development.html @@ -0,0 +1,133 @@ + + + + + + + + Using FedCloud client in Python — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Using FedCloud client in Python

+

FedCloud client can be used as a library for developing other services and tools for EGI Federated Cloud. Most of +functionalities of FedCloud client can be called directly from other codes without side effects. An example of the code +using FedCloud client is available at GitHub. +Just copy/download the code, add your access token and execute “python demo.py” to see how it works.

+
# Import FedCloud client library
+from fedcloudclient.openstack import fedcloud_openstack
+import json
+
+# Setting values for input parameters: token, site, VO
+token = "YOUR_ACCESS_TOKEN"
+site = "CYFRONET-CLOUD"
+vo = "fedcloud.egi.eu"
+
+# OpenStack command and options. Must be a tuple
+command = ("image", "list", "--long")
+
+# Execute the OpenStack command on the site/VO with single line of code
+# If command finishes correctly, the error_code is 0 and the result is stored
+# in JSON object for easy processing
+
+error_code, result = fedcloud_openstack(token, site, vo, command)
+
+# Check error code and print the result if OK
+if error_code == 0:
+    print(json.dumps(result, indent=4))
+else:
+    # If error, result is string containing error message
+    print("Error message is %s" % result)
+
+
+

Read the FedCloud client API references +for more details about each function in FedCloud client library. Check the output of the equivalent command of +FedCloud client and its source code to see how the function is used.

+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/fedcloudclient.html b/docs/docs/fedcloudclient.html new file mode 100644 index 0000000..31529e1 --- /dev/null +++ b/docs/docs/fedcloudclient.html @@ -0,0 +1,115 @@ + + + + + + + + FedCloud client API references — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

FedCloud client API references

+
+

fedcloudclient.checkin module

+
+
+

fedcloudclient.endpoint module

+
+
+

fedcloudclient.sites module

+
+
+

fedcloudclient.openstack module

+
+
+

fedcloudclient.cli module

+
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/index.html b/docs/docs/index.html new file mode 100644 index 0000000..b2f6279 --- /dev/null +++ b/docs/docs/index.html @@ -0,0 +1,150 @@ + + + + + + + + fedcloudclient documentation — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

fedcloudclient documentation

+

Add your content using reStructuredText syntax. See the +reStructuredText +documentation for details.

+
+

Contents:

+ +
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/install.html b/docs/docs/install.html new file mode 100644 index 0000000..3d1c152 --- /dev/null +++ b/docs/docs/install.html @@ -0,0 +1,172 @@ + + + + + + + + Installation — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Installation

+
+

Installing FedCloud client with pip

+

Simply use the following pip3 command (should be done without root privileges).

+
$ pip3 install -U fedcloudclient
+
+
+
+

Installation Notes

+

Installing the latest version of the FedCloud client package using pip3 will also install all required dependencies (such as openstackclient). It will create the executable files ``fedcloud`` and ``openstack``, placing them in the appropriate directory based on your Python environment:

+
    +

  • $VIRTUAL_ENV/bin – when using pip3 inside a Python virtual environment,

    • +

  • ~/.local/bin – when installing with pip3 --user,

    • +

  • /usr/local/bin – when installing system-wide with pip3 as root.

    • +
+
+

Note

+

If you install the package with the --user option, make sure that ~/.local/bin is included in your $PATH.

+
+
+
+

Verifying the Installation

+

To check if the installation was successful, run the following command:

+
::
+
+
+
+

$ fedcloud –help

+
+

This will show output:

+
::
+
+
+
+

Usage: fedcloud [OPTIONS] COMMAND [ARGS]…

+

CLI main function. Intentionally empty

+

Options: +–version Show the version and exit. +–help Show this message and exit.

+

Commands: +ec3 EC3 cluster provisioning +endpoint Obtain endpoint details and scoped tokens +openstack Execute OpenStack commands on site and VO +openstack-int Interactive OpenStack client on site and VO +secret Commands for accessing secret objects +select Select resources according to specification +site Obtain site configurations +token Get details of access token

+
+
+
+
+

Installing EGI Core Trust Anchor certificates

+

Some sites use certificates issued by national certificate authorities that are not included in the default +OS distribution. If you receive error message “SSL exception connecting to https:// …”, +follow instructions +for installing EGI Core Trust Anchor certificates and add them to the certificate bundle of Python requests. For quick +test in virtual environment, just execute the following commands. See this +README.md for more details.

+
$ wget https://raw.githubusercontent.com/tdviet/python-requests-bundle-certs/main/scripts/install_certs.sh
+$ bash install_certs.sh
+
+
+
+
+

Using FedCloud client via Docker container

+

You can use Docker container for testing FedCloud client without installation. EGI Core Trust Anchor certificates +and site configurations are preinstalled.

+
$ sudo docker pull tdviet/fedcloudclient
+$ sudo docker run -it  tdviet/fedcloudclient bash
+
+
+
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/intro.html b/docs/docs/intro.html new file mode 100644 index 0000000..ce8b5f2 --- /dev/null +++ b/docs/docs/intro.html @@ -0,0 +1,141 @@ + + + + + + + + Introduction — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Introduction

+https://zenodo.org/badge/336671726.svg + +

The FedCloud client is a high-level Python package for a command-line client +designed for interaction with the OpenStack services in the EGI infrastructure. The client can access various EGI +services and can perform many tasks for users including managing access tokens, listing services, and mainly execute +commands on OpenStack sites in EGI infrastructure.

+

The most notable features of FedCloud client are following:

+
    +

  • Rich functionalities: have wide ranges of useful commands, including checking access token, searching for +services, listing sites and VOs, and interaction with OpenStack sites.

    • +

  • Simple usages: can perform any OpenStack command on any sites with only three parameters: the site, the VO +and the command. For example, to list virtual machines (VM) images available to members of vo.access.egi.eu VO +on IISAS-FedCloud site, run the following command:

    • +
+
$ fedcloud openstack image list --vo vo.access.egi.eu --site IISAS-FedCloud
+
+
+
    +

  • Federation-wide: Single client for all OpenStack sites and related services of EGI Cloud infrastructure. +Single command may perform an action on all sites by specifying --site ALL_SITES.

    • +

  • Programmable: the client is designed for using in +scripts for automation +or as a Python library +for programming FedCloud services.

    • +
+

The following modules are included:

+
    +

  • fedcloudclient.conf for handling fedcloudclient configuration,

    • +

  • fedcloudclient.checkin for operation with EGI Check-in like getting tokens,

    • +

  • fedcloudclient.endpoint for searching endpoints via GOCDB, getting unscoped/scoped token from +OpenStack keystone,

    • +

  • fedcloudclient.sites for managing site configurations,

    • +

  • fedcloudclient.openstack for performing OpenStack operations on sites,

    • +

  • fedcloudclient.secret for accessing secrets in +Secret management service,

    • +

  • fedcloudclient.ec3 for deploying elastic computing clusters in Cloud.

    • +
+

A short tutorial of the fedcloudclient is available in this +presentation. +The full documentation, including installation, usage and API description is available +at https://fedcloudclient.fedcloud.eu/.

+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/modules.html b/docs/docs/modules.html new file mode 100644 index 0000000..eff15e3 --- /dev/null +++ b/docs/docs/modules.html @@ -0,0 +1,112 @@ + + + + + + + + fedcloudclient API references — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

fedcloudclient API references

+
+ +
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/quickstart.html b/docs/docs/quickstart.html new file mode 100644 index 0000000..e14ba32 --- /dev/null +++ b/docs/docs/quickstart.html @@ -0,0 +1,215 @@ + + + + + + + + Quick start — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Quick start

+

The Tutorial +presentation is designed for new users of FedCloud client. It starts with the quick setup and basic usages, +then step by step to more advanced scenarios.

+
+

Setup

+
    +

  • Install FedCloud client via pip:

    • +
+
$ pip3 install fedcloudclient
+
+
+

or use Docker container:

+
$ docker run -it  tdviet/fedcloudclient bash
+
+
+ +
$ export FEDCLOUD_OIDC_ACCESS_TOKEN=<ACCESS_TOKEN>
+
+
+
+
+

Basic usages

+
    +

  • List your VO memberships according to the access token:

    • +
+
$ fedcloud token list-vos
+eosc-synergy.eu
+fedcloud.egi.eu
+training.egi.eu
+...
+
+
+
    +

  • List sites in the EGI Federated Cloud

    • +
+
$ fedcloud site list
+100IT
+BIFI
+CESGA
+...
+
+
+
    +

  • List sites supporting a Virtual Organization in the EGI Federated Cloud

    • +
+
$ fedcloud site list --vo vo.access.egi.eu
+BIFI
+CENI
+CESGA-CLOUD
+...
+
+
+
    +

  • Execute an OpenStack command, e.g. list images in eosc-synergy.eu VO on IISAS-FedCloud site +(or other combination of site and VO you have access):

    • +
+
$ fedcloud openstack image list --site IISAS-FedCloud --vo eosc-synergy.eu
+Site: IISAS-FedCloud, VO: eosc-synergy.eu
++--------------------------------------+-------------------------------------------------+--------+
+| ID                                   | Name                                            | Status |
++--------------------------------------+-------------------------------------------------+--------+
+| 862d4ede-6a11-4227-8388-c94141a5dace | Image for EGI CentOS 7 [CentOS/7/VirtualBox]    | active |
+...
+
+
+
    +

  • Execute an OpenStack command, e.g. list VMs in eosc-synergy.eu VO on all sites +and print output in JSON format for further machine processing:

    • +
+
$ fedcloud openstack server list --site ALL_SITES --vo eosc-synergy.eu --json-output
+[
+{
+  "Site": "IISAS-FedCloud",
+  "VO": "eosc-synergy.eu",
+  "command": "server list",
+  "Exception": null,
+  "Error code": 0,
+  "Result": [
+    {
+      ...
+    },
+    ...
+  ]
+},
+...
+]
+
+
+
    +

  • Get helps from the client

    • +
+
$ fedcloud --help
+Usage: fedcloud [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  endpoint       Endpoint command group for interaction with GOCDB and endpoints
+  openstack      Executing OpenStack commands on site and VO
+  openstack-int  Interactive OpenStack client on site and VO
+  site           Site command group for manipulation with site configurations
+  token          Token command group for manipulation with tokens
+
+
+
    +

  • Read the Tutorial +presentation or next sections for more information.

    • +
+
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/scripts.html b/docs/docs/scripts.html new file mode 100644 index 0000000..3116615 --- /dev/null +++ b/docs/docs/scripts.html @@ -0,0 +1,179 @@ + + + + + + + + Using FedCloud client in scripts — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Using FedCloud client in scripts

+

FedCloud client can be used in scripts for simple automation, either for setting environment variables for other tools +or processing outputs from OpenStack commands.

+
+

Setting environment variables for external tools

+

Outputs from FedCloud client commands for setting environment variables are already in the forms “export VAR=VALUE”. +Simple eval command in scripts can be used for setting environment variables for external tools:

+
$ fedcloud site show-project-id --site IISAS-FedCloud --vo eosc-synergy.eu
+export OS_AUTH_URL="https://cloud.ui.savba.sk:5000/v3/"
+export OS_PROJECT_ID="51f736d36ce34b9ebdf196cfcabd24ee"
+
+# This command will set environment variables
+$ eval $(fedcloud site show-project-id --site IISAS-FedCloud --vo eosc-synergy.eu)
+
+# Check the value of the variable
+$ echo $OS_AUTH_URL
+https://cloud.ui.savba.sk:5000/v3/
+
+
+
+
+

Processing JSON outputs from OpenStack commands via jq

+

The outputs from Openstack command can be printed in JSON formats with –json-output parameter for further machine +processing. The JSON outputs can be processed in scripts by jq command. +For examples, if users want to select flavors with 2 CPUs:

+
$ fedcloud openstack flavor list  --site IISAS-FedCloud --vo eosc-synergy.eu --json-output
+[
+{
+  "Site": "IISAS-FedCloud",
+  "VO": "eosc-synergy.eu",
+  "command": "flavor list",
+  "Exception": null,
+  "Error code": 0,
+  "Result": [
+    {
+      "ID": "0",
+      "Name": "m1.nano",
+      "RAM": 64,
+      "Disk": 1,
+      "Ephemeral": 0,
+      "VCPUs": 1,
+      "Is Public": true
+    },
+    {
+      "ID": "2e562a51-8861-40d5-8fc9-2638bab4662c",
+      "Name": "m1.xlarge",
+      "RAM": 16384,
+      "Disk": 40,
+      "Ephemeral": 0,
+      "VCPUs": 8,
+      "Is Public": true
+    },
+    ...
+  ]
+}
+]
+
+# The following jq command selects flavors with VCPUs=2 and print their names
+$ fedcloud openstack flavor list  --site IISAS-FedCloud --vo eosc-synergy.eu --json-output | \
+    jq -r  '.[].Result[] | select(.VCPUs == 2) | .Name'
+m1.medium
+
+
+

The following example is more complex:

+
    +

  • List all flavors in the VO vo.access.egi.eu on all sites and print them in JSON format

    • +

  • Filter out sites with error code > 0

    • +

  • Select only GPU flavors

    • +

  • Filter out sites with empty list of GPU flavors

    • +

  • Print the result (list of all GPU flavors on all sites) in JSON format

    • +
+
$ fedcloud openstack flavor list --long --site ALL_SITES --vo vo.access.egi.eu --json-output | \
+    jq -r 'map(select(."Error code" ==  0)) |
+           map(.Result = (.Result| map(select(.Properties."Accelerator:Type" == "GPU")))) |
+           map(select(.Result | length >  0))'
+
+
+

Note that only OpenStack commands that have outputs can be used with –json-output. Using the parameter with +commands without outputs (e.g. setting properties) will generate errors of unsupported parameters.

+
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/docs/usage.html b/docs/docs/usage.html new file mode 100644 index 0000000..88eeb5f --- /dev/null +++ b/docs/docs/usage.html @@ -0,0 +1,671 @@ + + + + + + + + Usage — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Usage

+

FedCloud client has the following main groups of commands:

+
    +

  • “fedcloud config” for handling fedcloudclient configuration,

    • +

  • “fedcloud token” for interactions with EGI Check-in and access tokens,

    • +

  • “fedcloud endpoint” for interactions with GOCDB (and site endpoints according to GOCDB),

    • +

  • “fedcloud site” for manipulations with site configurations,

    • +

  • “fedcloud openstack” or “fedcloud openstack-int” for performing OpenStack commands on sites,

    • +

  • “fedcloud secret” for accessing secrets in +Secret management service,

    • +

  • “fedcloud ec3” as helper commands for deploying EC3.

    • +
+
+
+

Authentication Options

+

FedCloud commands require access tokens for authentication. Users have multiple options for providing these tokens:

+
    +

  • Direct access token: Use the --oidc-access-token option to provide an access token directly. You can retrieve this token from the environment variable FEDCLOUD_OIDC_ACCESS_TOKEN, or pass it explicitly, e.g.:

    +

    fedcloud token check --oidc-access-token <ACCESS_TOKEN>

    +
    • +

  • OIDC agent: Use the --oidc-agent-account option to integrate with oidc-agent. For example, check token validity with:

    +

    fedcloud token check --oidc-agent-account <NAME_OF_USER_FOR_OIDC_AGENT>

    +

    To use this method, follow the instructions at oidc-agent for EGI to register a client, then pass the client (account) name to the FedCloud client.

    +
    • +

  • Mytoken: Use the --mytoken option to authenticate with a token from the Mytoken service. To check token validity:

    +

    fedcloud token check --mytoken <TOKEN_FOR_MYTOKEN>

    +

    When creating a Mytoken, ensure you select “Allows obtaining OpenID Connect Access Tokens”. You may also use the --mytoken-server option to authenticate with a specific Mytoken server.

    +
    • +
+

Alternatively, you can obtain tokens using the EGI Check-in Token Portal, which provides all necessary information for EGI Check-in users.

+

In addition to command-line options, environment variables can be used for passing tokens, as summarized in the table below (not shown here).

+
+

By default, the protocol used is openid. This can be changed using the --os-protocol option. Note that some sites may have a fixed protocol defined in their site configuration (e.g., oidc for INFN-CLOUD-BARI).

+
+

Configuration

+

Display the current configuration of fedcloud with:

+
$ fedcloud config show
+
+
+

This will show a list of configuration parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

Default value

site

IISAS-FedCloud

vo

vo.access.egi.eu

site_list_url

https://raw.githubusercontent.com/tdviet/fedcloudclient/master/config/sites.yaml

site_dir

${HOME}/.config/fedcloud/site-config

oidc_url

https://aai.egi.eu/auth/realms/egi

gocdb_public_url

https://goc.egi.eu/gocdbpi/public/

gocdb_service_group

org.openstack.nova

vault_endpoint

https://vault.services.fedcloud.eu:8200

vault_role

vault_mount_point

/secrets/

vault_locker_mount_point

/v1/cubbyhole/

vault_salt

fedcloud_salt

log_file

${HOME}/.config/fedcloud/logs/fedcloud.log

log_level

DEBUG

log_config_file

${HOME}/.config/fedcloud/logging.conf

requests_cert_file

${HOME}/.config/fedcloud/cert/certs.pem

oidc_agent_account

egi

min_access_token_time

30

mytoken_server

https://mytoken.data.kit.edu

os_protocol

openid

os_auth_type

v3oidcaccesstoken

os_identity_provider

egi.eu

+

The FedCloud client supports multiple types of configuration:

+
    +

  • Default settings – accessible using ``DEFAULT_SETTINGS``, these are the built-in default values.

    • +

  • Local environment settings – custom configuration values defined in the environment and loaded via ``env_config``.

    • +

  • Saved configuration settings – user-defined settings stored in a JSON file, accessible via ``saved_config``.

    • +
+

For example, to print the environment configuration, use the following command:

+
$ fedcloud config show --source env_config
+
+
+

This command shows, for instance, the following output:

+
+
::

parameter value +—————— ——- +oidc_agent_account jaro221

+
+
+

The fedcloud configuration can be saved to a file using the following command

+
+
::

$ fedcloud config create

+
+
+

By default, the configuration file is saved to ${HOME}/.config/fedcloud/config.yaml, +but this location can be changed using the --config option. For example:

+
$ fedcloud config create --config-file /path/to/file.yaml
+
+
+
+
+
+

Using Environment Variables and Configuration Priorities

+

It is also possible to use the FEDCLOUD_CONFIG_FILE environment variable instead of the --config option in the command line. +This allows users to manage and switch between multiple configuration files—one per project—each with its own settings.

+

The fedcloud client supports configuration from multiple sources, in the following order of priority (highest to lowest):

+
    +

  1. Command-line options – override all other settings. +Example: --site IISAS-FedCloud

    2. +

  2. Environment variables – must begin with the prefix FEDCLOUD_. +Example: FEDCLOUD_SITE=IISAS-FedCloud

    3. +

  3. Configuration file – typically stored as config.yaml. +Example: the site setting in config.yaml

    4. +

  4. Default configuration – hardcoded defaults (lowest priority). +See the source code for details.

    5. +
+

The priority order is important: +default values are overridden by the configuration file, +which is overridden by environment variables, +which are in turn overridden by command-line options.

+

For example, the default configuration includes:

+
    +

  • site = IISAS-FedCloud

    • +

  • vo = vo.access.egi.eu

    • +
+

These values can be changed using any of the higher-priority methods. For example:

+
$ fedcloud openstack --vo training.egi.eu --site IFCA-LCG2 server list
+
+
+

or

+
$ export FEDCLOUD_VO=training.egi.eu
+$ export FEDCLOUD_SITE=IFCA-LCG2
+$ fedcloud openstack server list
+
+
+
+
+

Consistent Parameter Naming

+

Note the consistent naming convention for configuration parameters across different sources. For example, the same parameter is represented as:

+
    +

  • --oidc-agent-account in the command-line

    • +

  • FEDCLOUD_OIDC_AGENT_ACCOUNT as an environment variable

    • +

  • oidc_agent_account in the configuration file

    • +
+

All configuration parameters follow this consistent mapping across command-line options, environment variables, and configuration files.

+
+
+

Additional Configurable Parameters

+

In addition to oidc_agent_account, the following parameters can also be configured in the same way:

+
    +

  • site – the OpenStack site to target

    • +

  • vo – the Virtual Organisation (VO)

    • +

  • check_in_url – the EGI Check-in OIDC endpoint

    • +

  • client_id – the OIDC client ID

    • +

  • scopes – requested OIDC scopes

    • +

  • access_token – manually provided access token

    • +

  • output_format – format of output, e.g., table, json, or yaml

    • +
+

These parameters can be specified via:

+
    +

  • command-line options (e.g., --site, --vo),

    • +

  • environment variables (e.g., FEDCLOUD_SITE, FEDCLOUD_VO), or

    • +

  • configuration files (e.g., site: IISAS-FedCloud in config.yaml).

    • +
+

This design allows flexible and convenient configuration for various usage scenarios.

+ + + + + + + + + + + + + + + + + +

Environment variable

Command-line option

FEDCLOUD_OIDC_ACCESS_TOKEN

–oidc-access-token

FEDCLOUD_MYTOKEN

–mytoken

FEDCLOUD_OIDC_AGENT_ACCOUNT

–oidc-agent-account

+

For convenience, it is recommended to set transient parameters—such as access tokens—via environment variables. +This simplifies the usage of fedcloud commands by avoiding the need to specify these parameters on the command line each time.

+
+

Shell completion

+

Shell completion for fedcloud command in bash can be activated by executing the following command:

+
$ eval "$(_FEDCLOUD_COMPLETE=bash_source fedcloud)"
+
+
+

The command above may affect responsiveness of the shell. For long work, it is recommended to copy the +fedcloud_bash_completion.sh script to a local file, and +source it from ~/.bashrc. Refer Click documentation for a long explanation.

+

After enabling shell completion, press <TAB> twice for shell completion:

+
$ fedcloud site <TAB><TAB>
+env              list             save-config      show             show-project-id
+
+
+
+
+

fedcloud –help command

+
    +

  • “fedcloud –help” command will print help message. When using it in combination with other +commands, e.g. “fedcloud token –help”, “fedcloud token check –hep”, it will print list of options for the +corresponding commands

    • +
+
$ fedcloud --help
+Usage: fedcloud [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  config         Managing fedcloud configurations
+  endpoint       Obtain endpoint details and scoped tokens
+  openstack      Execute OpenStack commands on site and VO
+  openstack-int  Interactive OpenStack client on site and VO
+  secret         Commands for accessing secret objects
+  select         Select resources according to specification
+  site           Obtain site configurations
+  token          Get details of access token
+
+
+
+
+

fedcloud token commands

+
    +

  • ``fedcloud token check`` – Checks the expiration time of the configured access token, +allowing users to determine whether it needs to be refreshed.

    • +
+

As mentioned earlier, the access token can be provided via the environment variable FEDCLOUD_OIDC_ACCESS_TOKEN. +For this reason, the --oidc-access-token option is not shown in all examples below, even though it may be required if the token is not set via environment variables.

+
$ fedcloud token check
+
+
+

Output is shown as:

+
Access token is valid to 2021-01-02 01:25:39 UTC
+Access token expires in 3571 seconds
+
+
+
    +

  • “fedcloud token list-vos” : Print the list of VO memberships according to EGI Check-in

    • +
+
$ fedcloud token list-vos
+
+
+

Sample output:

+
eosc-synergy.eu
+fedcloud.egi.eu
+training.egi.eu
+
+
+
    +

  • “fedcloud token issue” : Print the access_token

    • +
+
$ fedcloud token issue
+
+
+

Sample output:

+
egwergwregrwegreg...
+
+
+
+
+

fedcloud endpoint commands

+

“fedcloud endpoint” commands are complementary part of the “fedcloud site” commands. Instead of using site +configurations defined in files saved in GitHub repository or local disk, the commands try to get site information +directly from GOCDB (Grid Operations Configuration Management Database) https://goc.egi.eu/ or make probe test on sites

+
    +

  • “fedcloud endpoint list” : List of endpoints of sites defined in GOCDB.

    • +
+
$ fedcloud endpoint list
+
+
+

Sample output:

+
+
::

Site type URL +—————— —————— ———————————————— +IFCA-LCG2 org.openstack.nova https://api.cloud.ifca.es:5000/v3/ +IN2P3-IRES org.openstack.nova https://sbgcloud.in2p3.fr:5000/v3 +…

+
+
+
    +

  • “fedcloud endpoint projects –site <SITE> –oidc-access-token <ACCESS_TOKEN>” : List of projects to which the owner +of the access token has access at the given site

    • +
+
$ fedcloud endpoint projects --site IFCA-LCG2
+id                                Name                        enabled    site
+--------------------------------  --------------------------  ---------  ---------
+2a7e2cd4b6dc4e609dd934964c1715c6  VO:demo.fedcloud.egi.eu     True       IFCA-LCG2
+3b9754ad8c6046b4aec43ec21abe7d8c  VO:eosc-synergy.eu          True       IFCA-LCG2
+...
+
+
+

If the site is set to ALL_SITES, or the argument -a is used, the command will show accessible projects from all sites of the EGI Federated Cloud.

+
    +
  • +
    “fedcloud endpoint vos –site <SITE> –oidc-access-token <ACCESS_TOKEN>”List of Virtual Organisations (VOs)

    to which the owner of the access token has access at the given site

    +
    +
    +
    • +
+
$ fedcloud endpoint vos --site IFCA-LCG2
+VO                id                                Project name         enabled    site
+----------------  --------------------------------  -------------------  ---------  ---------
+vo.access.egi.eu  233f045cb1ff46842a15ebb33af69460  VO:vo.access.egi.eu  True       IFCA-LCG2
+training.egi.eu   d340308880134d04294097524eace710  VO:training.egi.eu   True       IFCA-LCG2
+...
+
+
+

If the site is set to ALL_SITES, or the argument -a is used, the command will show accessible VOs from all sites of the EGI Federated Cloud.

+
$ fedcloud endpoint vos -a
+VO                   id                                Project name         enabled    site
+-------------------  --------------------------------  -------------------  ---------  -----------------
+vo.access.egi.eu     233f045cb1ff46842a15ebb33af69460  VO:vo.access.egi.eu  True       IFCA-LCG2
+training.egi.eu      d340308880134d04294097524eace710  VO:training.egi.eu   True       IFCA-LCG2
+vo.access.egi.eu     7101022b9ae74ed9ac1a574497279499  EGI_access           True       IN2P3-IRES
+vo.access.egi.eu     5bbdb5c1e0b2bcbac29904f4ac22dcaa  vo_access_egi_eu     True       UNIV-LILLE
+vo.access.egi.eu     4cab325ca8c2495bf2d4e8f230bcd51a  VO:vo.access.egi.eu  True       INFN-PADOVA-STACK
+...
+
+
+
    +

  • “fedcloud endpoint token –site <SITE> –project-id <PROJECT> –oidc-access-token <ACCESS_TOKEN>” : Get +OpenStack keystone scoped token on the site for the project ID.

    • +
+
$ fedcloud endpoint token --site IFCA-LCG2 --project-id 3b9754ad8c6046b4aec43ec21abe7d8c
+export FEDCLOUD_OS_TOKEN="gAAAAA..."
+
+
+
    +

  • “fedcloud endpoint env –site <SITE> –project-id <PROJECT> –oidc-access-token <ACCESS_TOKEN>” : Print +environment variables for working with the project ID on the site.

    • +
+
$ fedcloud endpoint env --site IFCA-LCG2 --project-id 3b9754ad8c6046b4aec43ec21abe7d8c
+# environment for IFCA-LCG2
+export FEDCLOUD_OS_AUTH_URL="https://api.cloud.ifca.es:5000/v3/"
+export FEDCLOUD_OS_AUTH_TYPE="v3oidcaccesstoken"
+export FEDCLOUD_OS_IDENTITY_PROVIDER="egi.eu"
+export FEDCLOUD_OS_PROTOCOL="openid"
+export FEDCLOUD_OS_ACCESS_TOKEN="..."
+
+
+
+
+

fedcloud site commands

+

“fedcloud site” commands will read site configurations and manipulate with them. If the local site configurations +exist at ~/.config/fedcloud/site-config/, fedcloud will read them from there, otherwise the commands will read +from GitHub repository.

+

By default, fedcloud does not save anything on local disk, users have to save the site configuration to local disk +explicitly via “fedcloud site save-config” command. The advantage of having local +site configurations, beside faster loading, is to give users ability to make customizations, e.g. add additional VOs, +remove sites they do not have access, and so on.

+
    +

  • “fedcloud site save-config” : Read the default site configurations from GitHub +and save them to ~/.config/fedcloud/site-config/ local directory. The command will overwrite existing site configurations +in the local directory.

    • +
+
$ fedcloud site save-config
+Saving site configs to directory /home/viet/.config/fedcloud/site-config/
+
+
+

After saving site configurations, users can edit and customize them, e.g. remove inaccessible sites, add new +VOs and so on.

+
    +

  • “fedcloud site list” : List of existing sites in the site configurations

    • +
+
$ fedcloud site list
+100IT
+BIFI
+CESGA
+...
+
+
+
    +

  • “fedcloud site list –vo <VO-name>” : List all sites supporting a Virtual Organization

    • +
+
$ fedcloud site vo-list --vo vo.access.egi.eu
+BIFI
+CENI
+CESGA-CLOUD
+...
+
+
+
    +

  • “fedcloud site show –site <SITE>” : Show configuration of the corresponding site.

    • +
+
$ fedcloud site show --site IISAS-FedCloud
+endpoint: https://cloud.ui.savba.sk:5000/v3/
+gocdb: IISAS-FedCloud
+vos:
+- auth:
+    project_id: a22bbffb007745b2934bf308b0a4d186
+  name: covid19.eosc-synergy.eu
+- auth:
+    project_id: 51f736d36ce34b9ebdf196cfcabd24ee
+  name: eosc-synergy.eu
+
+
+
    +

  • “fedcloud site show-project-id –site <SITE> –vo <VO>”: show the project ID of the VO on the site.

    • +
+
$ fedcloud site show-project-id --site IISAS-FedCloud --vo eosc-synergy.eu
+export FEDCLOUD_OS_AUTH_URL="https://cloud.ui.savba.sk:5000/v3/"
+export FEDCLOUD_OS_PROJECT_ID="51f736d36ce34b9ebdf196cfcabd24ee"
+
+
+
    +

  • “fedcloud site env –site <SITE> –vo <VO>”: set OpenStack environment variable for the VO on the site.

    • +
+
$ fedcloud site env --site IISAS-FedCloud --vo eosc-synergy.eu
+export FEDCLOUD_OS_AUTH_URL="https://cloud.ui.savba.sk:5000/v3/"
+export FEDCLOUD_OS_AUTH_TYPE="v3oidcaccesstoken"
+export FEDCLOUD_OS_IDENTITY_PROVIDER="egi.eu"
+export FEDCLOUD_OS_PROTOCOL="openid"
+export FEDCLOUD_OS_PROJECT_ID="51f736d36ce34b9ebdf196cfcabd24ee"
+# Remember to set OS_ACCESS_TOKEN, e.g. :
+# export FEDCLOUD_OS_ACCESS_TOKEN=`oidc-token egi`
+
+
+

The main differences between “fedcloud endpoint env” and “fedcloud site env” commands are that the second command +needs VO name as input parameter instead of project ID. The command may set also environment variable OS_ACCESS_TOKEN, +if access token is provided, otherwise it will print notification.

+
+
+

fedcloud select commands

+
    +

  • “fedcloud select flavor –site <SITE> –vo <VO> –oidc-access-token <ACCESS_TOKEN> –flavor-specs <flavor-specs>” : +Select flavor according to the specification in flavor-specs. The specifications may be repeated, +e.g. –flavor-specs “VCPUs==2” –flavor-specs “RAM>=2048”, or may be joined, e.g. +–flavor-specs “VCPUs==2 & Disk>10”. For frequently used specs, short-option alternatives are available, e.g. +–vcpus 2 is equivalent to –flavor-specs “VCPUs==2”. The output is sorted, flavors using less resources +(in the order: GPUs, CPUs, RAM, Disk) are placed on the first places. Users can choose to print only the best-matched +flavor with –output-format first (suitable for scripting) or the full list of all matched flavors in list/YAML/JSON +format.

    • +
+
$ fedcloud select flavor --site IISAS-FedCloud --vo vo.access.egi.eu --flavor-specs "RAM>=2096" --flavor-specs "Disk > 10" --output-format list
+m1.medium
+m1.large
+m1.xlarge
+m1.huge
+g1.c08r30-K20m
+g1.c16r60-2xK20m
+
+
+
    +

  • “fedcloud select image –site <SITE> –vo <VO> –oidc-access-token <ACCESS_TOKEN> –image-specs <image-specs>” : +Select image according to the specification in image-specs. The specifications may be repeated, +e.g. –image-specs “Name=~Ubuntu” –image-specs “Name=~’20.04’”. The output is sorted, newest images +are placed on the first places. Users can choose to print only the best-matched +image with –output-format first (suitable for scripting) or the full list of all matched images in list/YAML/JSON +format.

    • +
+
$ fedcloud select image --site INFN-CATANIA-STACK --vo training.egi.eu --image-specs "Name =~ Ubuntu" --output-format list
+TRAINING.EGI.EU Image for EGI Docker [Ubuntu/18.04/VirtualBox]
+TRAINING.EGI.EU Image for EGI Ubuntu 20.04 [Ubuntu/20.04/VirtualBox]
+
+
+
    +

  • “fedcloud select network –site <SITE> –vo <VO> –oidc-access-token <ACCESS_TOKEN> –network-specs <flavor-specs>” : +Select network according to the specification in network-specs. User can choose to select only public or private +network, or both (default). The output is sorted in the order: public, shared, +private. Users can choose to print only the best-matched network with –output-format first +(suitable for scripting) or the full list of all matched networks in list/YAML/JSON format.

    • +
+
$ fedcloud select network --site IISAS-FedCloud --vo training.egi.eu --network-specs default --output-format list
+public-network
+private-network
+
+
+
+
+

fedcloud openstack commands

+
    +

  • “fedcloud openstack –site <SITE> –vo <VO> –oidc-access-token <ACCESS_TOKEN> <OPENSTACK_COMMAND>” : Execute an +OpenStack command on the site and VO. Examples of OpenStack commands are “image list”, “server list” and can be used +with additional options for the commands, e.g. “image list –long”, “server list –format json”. The list of all +OpenStack commands, and their parameters/usages are available +here.

    • +
+
$ fedcloud openstack image list --site IISAS-FedCloud --vo eosc-synergy.eu
+Site: IISAS-FedCloud, VO: eosc-synergy.eu
++--------------------------------------+-------------------------------------------------+--------+
+| ID                                   | Name                                            | Status |
++--------------------------------------+-------------------------------------------------+--------+
+| 862d4ede-6a11-4227-8388-c94141a5dace | Image for EGI CentOS 7 [CentOS/7/VirtualBox]    | active |
+...
+
+
+

If the site is ALL_SITES, the OpenStack command will be executed on all sites in EGI Federated Cloud.

+
    +

  • “fedcloud openstack-int –site <SITE> –vo <VO> –oidc-access-token <ACCESS_TOKEN>” : Call OpenStack client without +command, so users can work with OpenStack site in interactive mode. This is useful when users need to perform multiple +commands successively. For example, users may need get list of images, list of flavors, list of networks before +creating a VM. OIDC authentication is done only once at the beginning, then the keystone token is cached and will +be used for successive commands without authentication via CheckIn again.

    • +
+
$ fedcloud openstack-int --site IISAS-FedCloud --vo eosc-synergy.eu
+(openstack) image list
++--------------------------------------+-------------------------------------------------+--------+
+| ID                                   | Name                                            | Status |
++--------------------------------------+-------------------------------------------------+--------+
+| 862d4ede-6a11-4227-8388-c94141a5dace | Image for EGI CentOS 7 [CentOS/7/VirtualBox]    | active |
+...
+(openstack) flavor list
++--------------------------------------+-----------+-------+------+-----------+-------+-----------+
+| ID                                   | Name      |   RAM | Disk | Ephemeral | VCPUs | Is Public |
++--------------------------------------+-----------+-------+------+-----------+-------+-----------+
+| 5bd8397c-b97f-462d-9d2b-5b533844996c | m1.small  |  2048 |   10 |         0 |     1 | True      |
+| df25f80f-ed19-4e0b-805e-d34620ba0334 | m1.medium |  4096 |   40 |         0 |     2 | True      |
+...
+(openstack)
+
+
+
+
+

fedcloud config commands

+
    +

  • “fedcloud config –config-file create” : Create default configuration file in default location for configuration file

    • +
+
+
+

fedcloud secret commands

+

The “fedcloud secret” commands are described in details in the documentation of the +Secret management service.

+
+
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/fedcloudclient-logo-non-transparent-small.png b/docs/fedcloudclient-logo-non-transparent-small.png deleted file mode 100644 index 74e50c3..0000000 Binary files a/docs/fedcloudclient-logo-non-transparent-small.png and /dev/null differ diff --git a/docs/genindex.html b/docs/genindex.html new file mode 100644 index 0000000..829eb3b --- /dev/null +++ b/docs/genindex.html @@ -0,0 +1,99 @@ + + + + + + + Index — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ + +

Index

+ +
+ +
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + +
+ + + + + + \ No newline at end of file diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..4de6be1 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,105 @@ + + + + + + + + fedcloudclient documentation — fedcloudclient documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

fedcloudclient documentation

+

Add your content using reStructuredText syntax. See the +reStructuredText +documentation for details.

+
+
+
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 620e4ed..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,42 +0,0 @@ -.. fedcloudclient documentation master file, created by - sphinx-quickstart on Sun Dec 27 22:25:01 2020. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -.. raw:: html - - - -Welcome to FedCloud client's documentation! -=========================================== - -.. image:: https://zenodo.org/badge/336671726.svg - :target: https://zenodo.org/badge/latestdoi/336671726 - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - intro - quickstart - install - usage - development - scripts - fedcloudclient - FAQ - cheat - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/install.rst b/docs/install.rst deleted file mode 100644 index 8644e6d..0000000 --- a/docs/install.rst +++ /dev/null @@ -1,52 +0,0 @@ -Installation -============ - -Installing FedCloud client with pip -*********************************** - -Simply use the following **pip3** command (should be done without root privileges). - -:: - - $ pip3 install -U fedcloudclient - -That will install latest version **FedCloud client** package together with its required packages -(like **openstackclient**). It will also create executable files **"fedcloud"** and **"openstack"** and add them to -corresponding directory according to your Python execution environment (*$VIRTUAL_ENV/bin* for executing *pip3* in -Python virtual environment, *~/.local/bin* for executing *pip3* as user (with *--user* option), and */usr/local/bin* -when executing *pip3* as root). Make sure to add *~/.local/bin* to $PATH if installing as user. - -Check if the installation is correct by executing the client - -:: - - $ fedcloud - -Installing EGI Core Trust Anchor certificates -********************************************* - -Some sites use certificates issued by national certificate authorities that are not included in the default -OS distribution. If you receive error message *"SSL exception connecting to https:// ..."*, -follow `instructions `_ -for installing EGI Core Trust Anchor certificates and add them to the certificate bundle of Python requests. For quick -test in virtual environment, just execute the following commands. See this -`README.md `_ for more details. - -:: - - $ wget https://raw.githubusercontent.com/tdviet/python-requests-bundle-certs/main/scripts/install_certs.sh - $ bash install_certs.sh - -Using FedCloud client via Docker container -****************************************** - -You can use Docker container for testing **FedCloud client** without installation. EGI Core Trust Anchor certificates -and site configurations are preinstalled. - -:: - - $ sudo docker pull tdviet/fedcloudclient - $ sudo docker run -it tdviet/fedcloudclient bash - - - diff --git a/docs/objects.inv b/docs/objects.inv new file mode 100644 index 0000000..3759d23 --- /dev/null +++ b/docs/objects.inv @@ -0,0 +1,5 @@ +# Sphinx inventory version 2 +# Project: fedcloudclient +# Version: +# The remainder of this file is compressed using zlib. +xڍMn >Hn"Eʢ 01\~XJwߛa?+7^/ړ"ɤkEx.boج"}#F-w!b%r~P&p”LԤya4&^dw7Xa~bȪnCzY;yFH1愗ـZxq7 T͆>;Hg NcVN6Z^aTpt&(5&%E/۪Yc,{(1v + + + + + + Search — fedcloudclient documentation + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Search

+ + + + +

+ Searching for multiple words only shows matches that contain + all words. +

+ + +
+ + + +
+ + +
+ + +
+ +
+
+ +
+
+
+ ©2025, jaro221. + + | + Powered by Sphinx 8.1.3 + & Alabaster 1.0.0 + +
+ + + + + + \ No newline at end of file diff --git a/docs/searchindex.js b/docs/searchindex.js new file mode 100644 index 0000000..8c276d5 --- /dev/null +++ b/docs/searchindex.js @@ -0,0 +1 @@ +Search.setIndex({"alltitles": {"Additional Configurable Parameters": [[10, "additional-configurable-parameters"]], "Authentication Options": [[10, "authentication-options"]], "Basic usages": [[1, "basic-usages"], [8, "basic-usages"]], "By default, the protocol used is openid. This can be changed using the --os-protocol option. Note that some sites may have a fixed protocol defined in their site configuration (e.g., oidc for INFN-CLOUD-BARI).": [[10, "by-default-the-protocol-used-is-openid-this-can-be-changed-using-the-os-protocol-option-note-that-some-sites-may-have-a-fixed-protocol-defined-in-their-site-configuration-e-g-oidc-for-infn-cloud-bari"]], "Cheat sheet": [[1, null]], "Configuration": [[10, "configuration"]], "Consistent Parameter Naming": [[10, "consistent-parameter-naming"]], "Contents:": [[4, null]], "FAQ and Troubleshooting": [[0, null]], "FedCloud client API references": [[3, null]], "Installation": [[5, null]], "Installation Notes": [[5, "installation-notes"]], "Installing EGI Core Trust Anchor certificates": [[5, "installing-egi-core-trust-anchor-certificates"]], "Installing FedCloud client with pip": [[5, "installing-fedcloud-client-with-pip"]], "Introduction": [[6, null]], "Local install via pip3": [[1, "local-install-via-pip3"]], "Mapping and filtering results from OpenStack commands": [[1, "mapping-and-filtering-results-from-openstack-commands"]], "More information": [[1, "more-information"]], "Processing JSON outputs from OpenStack commands via jq": [[9, "processing-json-outputs-from-openstack-commands-via-jq"]], "Quick start": [[8, null]], "Searching and selecting resources": [[1, "searching-and-selecting-resources"]], "Setting environment variables for external tools": [[9, "setting-environment-variables-for-external-tools"]], "Setup": [[8, "setup"]], "Shell completion": [[10, "shell-completion"]], "Usage": [[10, null]], "Useful commands": [[1, "useful-commands"]], "Using Docker container": [[1, "using-docker-container"]], "Using Environment Variables and Configuration Priorities": [[10, "using-environment-variables-and-configuration-priorities"]], "Using FedCloud client in Python": [[2, null]], "Using FedCloud client in scripts": [[9, null]], "Using FedCloud client via Docker container": [[5, "using-fedcloud-client-via-docker-container"]], "Using oidc-agent": [[1, "using-oidc-agent"]], "Verifying the Installation": [[5, "verifying-the-installation"]], "fedcloud config commands": [[10, "fedcloud-config-commands"]], "fedcloud endpoint commands": [[10, "fedcloud-endpoint-commands"]], "fedcloud openstack commands": [[10, "fedcloud-openstack-commands"]], "fedcloud secret commands": [[10, "fedcloud-secret-commands"]], "fedcloud select commands": [[10, "fedcloud-select-commands"]], "fedcloud site commands": [[10, "fedcloud-site-commands"]], "fedcloud token commands": [[10, "fedcloud-token-commands"]], "fedcloud \u2013help command": [[10, "fedcloud-help-command"]], "fedcloudclient API references": [[7, null]], "fedcloudclient documentation": [[4, null], [11, null]], "fedcloudclient.checkin module": [[3, "fedcloudclient-checkin-module"]], "fedcloudclient.cli module": [[3, "fedcloudclient-cli-module"]], "fedcloudclient.endpoint module": [[3, "fedcloudclient-endpoint-module"]], "fedcloudclient.openstack module": [[3, "fedcloudclient-openstack-module"]], "fedcloudclient.sites module": [[3, "fedcloudclient-sites-module"]]}, "docnames": ["docs/FAQ", "docs/cheat", "docs/development", "docs/fedcloudclient", "docs/index", "docs/install", "docs/intro", "docs/modules", "docs/quickstart", "docs/scripts", "docs/usage", "index"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["docs/FAQ.rst", "docs/cheat.rst", "docs/development.rst", "docs/fedcloudclient.rst", "docs/index.rst", "docs/install.rst", "docs/intro.rst", "docs/modules.rst", "docs/quickstart.rst", "docs/scripts.rst", "docs/usage.rst", "index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"": 2, "0": [1, 2, 8, 9, 10], "01": 10, "02": 10, "04": [1, 10], "1": [9, 10], "10": 10, "100it": [8, 10], "16384": 9, "169": 1, "18": [1, 10], "2": [1, 9, 10], "20": [1, 10], "2021": 10, "2048": [1, 10], "2096": 10, "233f045cb1ff46842a15ebb33af69460": 10, "25": 10, "254": 1, "2638bab4662c": 9, "2a7e2cd4b6dc4e609dd934964c1715c6": 10, "2e562a51": 9, "2xk20m": 10, "30": 10, "300": 0, "3571": 10, "39": 10, "3b9754ad8c6046b4aec43ec21abe7d8c": 10, "4": 2, "40": [9, 10], "4096": 10, "40d5": 9, "4227": [8, 10], "462d": 10, "4cab325ca8c2495bf2d4e8f230bcd51a": 10, "4e0b": 10, "5000": [9, 10], "51f736d36ce34b9ebdf196cfcabd24e": [9, 10], "5b533844996c": 10, "5bbdb5c1e0b2bcbac29904f4ac22dcaa": 10, "5bd8397c": 10, "64": 9, "6a11": [8, 10], "7": [8, 10], "7101022b9ae74ed9ac1a574497279499": 10, "8": 9, "805e": 10, "8200": 10, "8388": [8, 10], "862d4ede": [8, 10], "8861": 9, "8fc9": 9, "9d2b": 10, "A": 6, "As": 10, "By": 4, "For": [0, 5, 6, 9, 10], "If": [0, 2, 5, 10], "In": 10, "It": [0, 5, 8, 10], "On": 0, "The": [0, 6, 8, 9, 10], "These": 10, "To": [5, 10], "_fedcloud_complet": [1, 10], "a22bbffb007745b2934bf308b0a4d186": 10, "aai": [1, 10], "abil": 10, "about": 2, "abov": 10, "acc": 1, "acceler": [1, 9], "access": [1, 2, 5, 6, 8, 9, 10], "access_token": [8, 10], "accord": [1, 5, 8, 10], "account": [1, 10], "across": 10, "action": 6, "activ": [1, 8, 10], "add": [0, 2, 4, 5, 10, 11], "addit": 4, "advanc": 8, "advantag": 10, "affect": 10, "after": [0, 10], "again": 10, "agent": [0, 4, 8, 10], "all": [1, 5, 6, 8, 9, 10], "all_sit": [1, 6, 8, 9, 10], "allow": 10, "alreadi": 9, "also": [0, 5, 10], "altern": 10, "an": [1, 2, 6, 8, 10], "anchor": [0, 4], "ani": [6, 10], "anyth": 10, "api": [2, 4, 6, 10], "appdb": 1, "applianc": 1, "appropri": 5, "ar": [0, 5, 6, 9, 10], "arg": [5, 8, 10], "argument": 10, "auth": [1, 10], "authent": 4, "author": [0, 5], "autom": [6, 9], "avail": [1, 2, 6, 10], "avoid": 10, "b97f": 10, "bari": 4, "base": 5, "bash": [0, 1, 5, 8, 10], "bash_sourc": [1, 10], "bashrc": 10, "basic": 4, "befor": 10, "begin": 10, "below": 10, "besid": 10, "best": 10, "between": 10, "bifi": [8, 10], "bin": [1, 5], "both": 10, "built": 10, "bundl": [0, 1, 5], "c": 1, "c08r30": 10, "c16r60": 10, "c94141a5dac": [8, 10], "cach": 10, "call": [2, 10], "can": [1, 2, 4, 5, 6, 9], "cat": [0, 1], "catania": 10, "ceni": [8, 10], "cento": [0, 8, 10], "cert": [0, 1, 5, 10], "certif": [0, 1, 4], "cesga": [1, 8, 10], "cesnet": 1, "chang": 4, "cheat": 4, "check": [0, 1, 2, 5, 6, 8, 9, 10], "check_in_url": 10, "checkin": [6, 7, 10], "choos": 10, "cli": [5, 7], "click": 10, "client": [0, 4, 6, 7, 8, 10], "client_id": 10, "cloud": [1, 2, 4, 6, 8, 9], "cluster": [5, 6], "code": [1, 2, 8, 9, 10], "com": [0, 1, 5, 10], "combin": [8, 10], "command": [0, 2, 4, 5, 6, 8], "comp": 1, "complementari": 10, "complet": 1, "complex": [1, 9], "comput": 6, "conf": [6, 10], "config": 1, "configur": [4, 5, 6, 8], "connect": [0, 5, 10], "consist": 4, "construct": 1, "contain": [2, 4, 8], "content": 11, "conveni": 10, "convent": 10, "copi": [1, 2, 10], "core": [0, 4], "correctli": 2, "correspond": 10, "covid19": 10, "cpu": [1, 9, 10], "creat": [1, 5, 10], "cubbyhol": 10, "curl": 1, "current": 10, "custom": 10, "cyfronet": 2, "d340308880134d04294097524eace710": 10, "d34620ba0334": 10, "daemon": 0, "dashboard": 1, "data": [1, 10], "databas": 10, "dc": 1, "debug": 10, "default": [0, 4, 5], "default_set": 10, "defin": 4, "demo": [2, 10], "depend": 5, "deploi": [6, 10], "describ": [0, 10], "descript": 6, "design": [6, 8, 10], "detail": [0, 1, 2, 4, 5, 10, 11], "determin": 10, "develop": 2, "df25f80f": 10, "differ": 10, "direct": 10, "directli": [2, 10], "directori": [5, 10], "dirti": 1, "disk": [9, 10], "displai": 10, "distribut": [0, 5], "do": 10, "docker": [4, 8, 10], "document": [6, 10], "doe": 10, "done": [1, 5, 10], "download": 2, "dump": 2, "dure": 0, "e": [4, 8, 9], "each": [2, 10], "earlier": 10, "easi": 2, "ec3": [5, 6, 10], "echo": 9, "ed19": 10, "edit": 10, "edu": [1, 10], "eduperson_entitl": 1, "effect": 2, "egi": [0, 1, 2, 4, 6, 8, 9, 10], "egi_access": 10, "egwergwregrwegreg": 10, "either": 9, "elast": 6, "els": 2, "email": 1, "empti": [5, 9], "enabl": 10, "endpoint": [1, 5, 6, 7, 8], "ensur": 10, "entropi": 0, "entropy_avail": 0, "env": [1, 10], "env_config": 10, "environ": [0, 1, 4, 5, 8], "eosc": [1, 8, 9, 10], "ephemer": [9, 10], "equival": [2, 10], "error": [0, 1, 2, 5, 8, 9], "error_cod": 2, "eu": [1, 2, 6, 8, 9, 10], "eval": [1, 9, 10], "even": 10, "exampl": [1, 2, 6, 9, 10], "except": [0, 5, 8, 9], "execut": [0, 1, 2, 5, 6, 8, 10], "exist": 10, "exit": [5, 8, 10], "expir": [1, 10], "explan": 10, "explicitli": 10, "export": [1, 8, 9, 10], "extern": 4, "f": 1, "faq": 4, "faster": 10, "featur": 6, "fedcloud": [0, 1, 4, 6, 7, 8], "fedcloud_": 10, "fedcloud_bash_complet": [1, 10], "fedcloud_config_fil": 10, "fedcloud_mytoken": [1, 10], "fedcloud_oidc_access_token": [8, 10], "fedcloud_oidc_agent_account": 10, "fedcloud_openstack": 2, "fedcloud_os_access_token": 10, "fedcloud_os_auth_typ": 10, "fedcloud_os_auth_url": 10, "fedcloud_os_identity_provid": 10, "fedcloud_os_project_id": 10, "fedcloud_os_protocol": 10, "fedcloud_os_token": 10, "fedcloud_salt": 10, "fedcloud_sit": [1, 10], "fedcloud_vo": [1, 10], "fedcloudcli": [1, 2, 5, 6, 8, 10], "feder": [1, 2, 6, 8, 10], "file": [1, 5, 10], "filter": [4, 9], "finish": 2, "first": 10, "fix": 4, "flavor": [1, 9, 10], "flexibl": 10, "follow": [0, 5, 6, 9, 10], "form": 9, "format": [1, 8, 9, 10], "fr": 10, "frequent": 10, "from": [2, 4, 6, 8, 10], "frozen": 0, "full": [1, 6, 10], "function": [2, 5, 6], "further": [8, 9], "g": [4, 8, 9], "g1": 10, "gaaaaa": 10, "gen": 1, "gener": 9, "get": [1, 5, 6, 8, 10], "github": [2, 10], "githubusercont": [0, 1, 5, 10], "give": [0, 10], "given": 10, "goc": 10, "gocdb": [6, 8, 10], "gocdb_public_url": 10, "gocdb_service_group": 10, "gocdbpi": 10, "gpu": [1, 9, 10], "grid": 10, "group": [8, 10], "ha": 10, "handl": [6, 10], "hardcod": 10, "have": [0, 4, 6, 8, 9], "haveg": 0, "help": [1, 5, 8], "helper": 10, "hep": 10, "here": [0, 10], "high": 6, "higher": 10, "highest": 10, "home": 10, "horizon": 1, "how": 2, "http": [0, 1, 5, 6, 9, 10], "huge": 10, "i": [0, 1, 2, 4, 5, 6, 8, 9], "id": [1, 8, 9, 10], "ifca": [1, 10], "igtf": 1, "iisa": [1, 6, 8, 9, 10], "imag": [1, 2, 6, 8, 10], "import": [2, 10], "in2p3": 10, "inaccess": 10, "includ": [0, 5, 6, 10], "indent": 2, "infn": 4, "inform": [4, 8, 10], "infrastructur": 6, "initi": 0, "input": [2, 10], "insid": 5, "instal": [0, 4, 6, 8], "install_cert": [0, 1, 5], "instanc": 10, "instead": 10, "instruct": [0, 5, 8, 10], "int": [5, 8, 10], "integr": 10, "intention": 5, "interact": [5, 6, 8, 10], "introduct": 4, "ir": 10, "issu": [0, 1, 5, 10], "issuer": 1, "its": [2, 10], "jaro221": 10, "join": 10, "jq": [1, 4], "json": [1, 2, 4, 8, 10], "just": [0, 1, 2, 5], "k20m": 10, "kei": 1, "kernel": 0, "keychain": 1, "keypair": 1, "keyston": [6, 10], "kit": [1, 10], "known": 0, "larg": 10, "latest": [1, 5], "lcg2": [1, 10], "length": [1, 9], "less": 10, "level": 6, "librari": [0, 2, 6], "libsodium": 0, "like": 6, "lill": 10, "line": [2, 6, 10], "list": [1, 2, 6, 8, 9, 10], "load": [1, 10], "local": [4, 5, 10], "locat": 10, "log": [1, 10], "log_config_fil": 10, "log_fil": 10, "log_level": 10, "long": [1, 2, 9, 10], "longer": 1, "lower": 0, "lowest": 10, "m": 1, "m1": [9, 10], "machin": [0, 1, 6, 8, 9], "mai": [1, 4, 6], "main": [0, 1, 5, 10], "mainli": [0, 6], "make": [5, 10], "manag": [6, 10], "mani": 6, "manipul": [8, 10], "manual": [0, 10], "map": [4, 9, 10], "master": [1, 10], "match": 10, "mcc": 1, "md": [0, 5], "medium": [9, 10], "member": 6, "membership": [1, 8, 10], "mention": 10, "messag": [0, 2, 5, 8, 10], "method": 10, "min_access_token_tim": 10, "mode": 10, "modul": [6, 7], "more": [0, 2, 4, 5, 8, 9], "most": [2, 6], "multipl": 10, "must": [2, 10], "my": 1, "mytoken": [1, 10], "mytoken_serv": 10, "name": [1, 4, 8, 9], "name_of_user_for_oidc_ag": 10, "nano": 9, "nation": [0, 5], "necessari": 10, "need": 10, "network": 10, "new": [8, 10], "newest": 10, "next": 8, "notabl": 6, "note": [4, 9], "notif": 10, "nova": 10, "null": [8, 9], "o": [0, 4, 5], "object": [1, 2, 5, 10], "obtain": [5, 10], "oidc": [0, 4, 8], "oidc_agent_account": [1, 10], "oidc_url": 10, "ok": 2, "onc": [1, 10], "one": 10, "onli": [1, 6, 9, 10], "openid": 4, "openstack": [2, 4, 5, 6, 7, 8], "openstack_command": 10, "openstackcli": 5, "oper": [6, 10], "option": [2, 4, 5, 8], "order": 10, "org": [1, 10], "organ": [1, 8, 10], "organis": 10, "os_access_token": 10, "os_auth_typ": 10, "os_auth_url": 9, "os_identity_provid": 10, "os_project_id": 9, "os_protocol": 10, "os_token": 1, "other": [1, 2, 8, 9, 10], "otherwis": 10, "out": 9, "output": [1, 2, 4, 5, 8, 10], "output_format": 10, "overrid": 10, "overridden": 10, "overwrit": 10, "own": [1, 10], "owner": 10, "packag": [5, 6], "padova": 10, "paramet": [2, 4, 6, 9], "part": 10, "pass": [1, 10], "path": [5, 10], "pem": 10, "per": 10, "perform": [6, 10], "pip": [1, 4, 8], "pip3": [4, 5, 8], "place": [5, 10], "portal": [8, 10], "possibl": 10, "prefix": 10, "preinstal": 5, "present": [6, 8], "press": 10, "previou": 1, "previous": 1, "print": [1, 2, 8, 9, 10], "prioriti": 4, "privat": 10, "privileg": 5, "probe": 10, "problem": 0, "proc": 0, "process": [2, 4, 8], "program": 6, "programm": 6, "project": [1, 9, 10], "project_id": 10, "properti": [1, 9], "protocol": 4, "provid": 10, "provis": 5, "pub": 1, "public": [9, 10], "pull": [1, 5], "py": 2, "python": [0, 1, 4, 5, 6], "python3": 1, "quick": [0, 1, 4, 5], "r": [1, 9], "ram": [1, 9, 10], "random": 0, "rang": 6, "raw": [0, 1, 5, 10], "read": [2, 8, 10], "readm": [0, 5], "realm": [1, 10], "reason": 10, "reboot": 0, "receiv": [0, 5], "recommend": 10, "refer": [2, 4, 10], "refresh": 10, "regex": 1, "regist": 10, "relat": 6, "rememb": 10, "remov": [1, 10], "repeat": 10, "repositori": 10, "repres": 10, "request": [0, 1, 5, 10], "requests_cert_fil": 10, "requir": [5, 10], "resourc": [4, 5, 10], "respons": 10, "restart": 1, "restructuredtext": [4, 11], "result": [0, 2, 4, 8, 9], "retriev": [1, 10], "rich": 6, "rng": 0, "root": [1, 5], "run": [1, 5, 6, 8], "same": 10, "sampl": 10, "savba": [9, 10], "save": 10, "saved_config": 10, "sbgcloud": 10, "scenario": [8, 10], "scope": [1, 5, 6, 10], "script": [0, 1, 4, 5, 6, 10], "search": [4, 6], "second": 10, "secret": [5, 6], "section": 8, "see": [0, 1, 2, 4, 5, 10, 11], "select": [4, 5, 9], "server": [1, 8, 10], "servic": [1, 2, 6, 10], "set": [1, 2, 4, 8, 10], "setup": 4, "sh": [0, 1, 5, 10], "share": 10, "sheet": 4, "shell": 1, "short": [6, 10], "shorter": 1, "should": 5, "show": [1, 5, 8, 9, 10], "shown": 10, "side": 2, "simpl": [6, 9], "simpler": 1, "simpli": 5, "simplifi": 10, "singl": [2, 6], "site": [0, 1, 2, 4, 5, 6, 7, 8, 9], "site_dir": 10, "site_list_url": 10, "sk": [9, 10], "small": 10, "so": 10, "some": [0, 4, 5], "sort": 10, "sourc": [1, 2, 10], "spec": [1, 10], "specif": [5, 10], "specifi": [6, 10], "ssl": [0, 5], "stack": 10, "start": [0, 1, 4], "statu": [8, 10], "step": 8, "store": [2, 10], "string": 2, "success": [5, 10], "sudo": [1, 5], "suitabl": 10, "summar": 10, "support": [1, 8, 10], "sure": 5, "switch": 10, "sy": 0, "synergi": [1, 8, 9, 10], "syntax": [4, 11], "system": 5, "systemat": 1, "tab": 10, "tabl": 10, "target": 10, "task": 6, "tdviet": [0, 1, 5, 8, 10], "termin": 1, "test": [0, 5, 10], "testvm": 1, "than": 0, "thei": 10, "them": [0, 5, 9, 10], "thi": [0, 4, 5, 6, 8, 9], "though": 10, "three": 6, "time": [1, 10], "titl": 1, "token": [1, 2, 5, 6, 8], "token_for_mytoken": 10, "tool": [0, 2, 4], "train": [1, 8, 10], "transient": 10, "troubleshoot": 4, "true": [9, 10], "trust": [0, 4], "try": 10, "tupl": 2, "turn": 10, "tutori": [1, 6, 8], "twice": 10, "txt": 1, "type": [1, 9, 10], "typic": 10, "u": 5, "ubuntu": [1, 10], "ui": [9, 10], "univ": 10, "unrespons": 1, "unscop": 6, "unsupport": 9, "url": 10, "us": [0, 4, 6, 8, 11], "usag": [4, 5, 6], "user": [1, 5, 6, 8, 9, 10], "user_data": 1, "usr": 5, "utc": 10, "v": 1, "v1": 10, "v3": [9, 10], "v3oidcaccesstoken": 10, "valid": 10, "valu": [1, 2, 9, 10], "var": 9, "variabl": [1, 4, 8], "variou": [6, 10], "vault": 10, "vault_endpoint": 10, "vault_locker_mount_point": 10, "vault_mount_point": 10, "vault_rol": 10, "vault_salt": 10, "vcpu": [1, 9, 10], "venv": 1, "version": [1, 5], "via": [4, 6, 8, 10], "viet": 10, "virtual": [0, 1, 5, 6, 8, 10], "virtual_env": 5, "virtualbox": [1, 8, 10], "vm": [0, 1, 6, 8, 10], "vo": [1, 2, 5, 6, 8, 9, 10], "vo_access_egi_eu": 10, "wa": 5, "wai": [1, 10], "want": 9, "wget": [0, 1, 5], "when": [0, 5, 10], "whether": 10, "which": [0, 10], "wide": [5, 6], "without": [2, 5, 9, 10], "work": [1, 2, 10], "xlarg": [9, 10], "yaml": 10, "you": [0, 1, 5, 8, 10], "your": [1, 2, 4, 5, 8, 11], "your_access_token": 2}, "titles": ["FAQ and Troubleshooting", "Cheat sheet", "Using FedCloud client in Python", "FedCloud client API references", "fedcloudclient documentation", "Installation", "Introduction", "fedcloudclient API references", "Quick start", "Using FedCloud client in scripts", "Usage", "fedcloudclient documentation"], "titleterms": {"By": 10, "addit": 10, "agent": 1, "anchor": 5, "api": [3, 7], "authent": 10, "bari": 10, "basic": [1, 8], "can": 10, "certif": 5, "chang": 10, "cheat": 1, "checkin": 3, "cli": 3, "client": [2, 3, 5, 9], "cloud": 10, "command": [1, 9, 10], "complet": 10, "config": 10, "configur": 10, "consist": 10, "contain": [1, 5], "content": 4, "core": 5, "default": 10, "defin": 10, "docker": [1, 5], "document": [4, 11], "e": 10, "egi": 5, "endpoint": [3, 10], "environ": [9, 10], "extern": 9, "faq": 0, "fedcloud": [2, 3, 5, 9, 10], "fedcloudcli": [3, 4, 7, 11], "filter": 1, "fix": 10, "from": [1, 9], "g": 10, "have": 10, "help": 10, "i": 10, "infn": 10, "inform": 1, "instal": [1, 5], "introduct": 6, "jq": 9, "json": 9, "local": 1, "mai": 10, "map": 1, "modul": 3, "more": 1, "name": 10, "note": [5, 10], "o": 10, "oidc": [1, 10], "openid": 10, "openstack": [1, 3, 9, 10], "option": 10, "output": 9, "paramet": 10, "pip": 5, "pip3": 1, "prioriti": 10, "process": 9, "protocol": 10, "python": 2, "quick": 8, "refer": [3, 7], "resourc": 1, "result": 1, "script": 9, "search": 1, "secret": 10, "select": [1, 10], "set": 9, "setup": 8, "sheet": 1, "shell": 10, "site": [3, 10], "some": 10, "start": 8, "thi": 10, "token": 10, "tool": 9, "troubleshoot": 0, "trust": 5, "us": [1, 2, 5, 9, 10], "usag": [1, 8, 10], "variabl": [9, 10], "verifi": 5, "via": [1, 5, 9]}}) \ No newline at end of file diff --git a/docs/usage.rst b/docs/usage.rst deleted file mode 100644 index 6b944c6..0000000 --- a/docs/usage.rst +++ /dev/null @@ -1,436 +0,0 @@ -Usage -===== - -**FedCloud client** has six main groups of commands: - -* **"fedcloud token"** for interactions with EGI Check-in and access tokens, - -* **"fedcloud endpoint"** for interactions with GOCDB (and site endpoints according to GOCDB), - -* **"fedcloud site"** for manipulations with site configurations, - -* **"fedcloud openstack"** or **"fedcloud openstack-int"** for performing OpenStack commands on sites, - -* **fedcloud secret** for accessing secrets in - `Secret management service `_, - -* **"fedcloud ec3"** as helper commands for deploying EC3. - - -Authentication -************** - -Many **fedcloud** commands need access tokens for authentication. Users can choose whether to provide access tokens -directly (via option *"--oidc-access-token"*), via `oidc-agent `_ -(via option *"--oidc-agent-account"*), or via `mytoken `_ (via option *"--mytoken"*). - -Users of EGI Check-in can get all information needed for obtaining access tokens from `EGI Check-in Token -Portal `_. For providing access token via *oidc-agent*, follow the instructions from -`oidc-agent `_ for registering a client, then -give the client name (account name in *oidc-agent*) to *FedCloud client* via option *"--oidc-agent-account"*. -On the other hand visit the `mytoken `_ website to configure a *mytoken*, -remember to check *"Allows obtaining OpenID Connect Access Tokens"*, and use the option *"--mytoken"* -to pass it to *FedCloud client"*. Environment variables can be use instead of the command-line options, -as explained in the table below. - -The default protocol is *"openid"*. Users can change default protocol via option *"--openstack-auth-protocol"*. However, -sites may have protocol fixedly defined in site configuration, e.g. *"oidc"* for INFN-CLOUD-BARI. - -Environment variables -********************* - -Most of fedcloud options, including options for tokens can be set via environment variables: - -+-----------------------------+---------------------------------+------------------------------------+ -| Environment variables | Command-line options | Default value | -+=============================+=================================+====================================+ -| OIDC_AGENT_ACCOUNT | --oidc-agent-account | | -+-----------------------------+---------------------------------+------------------------------------+ -| OIDC_ACCESS_TOKEN | --oidc-access-token | | -+-----------------------------+---------------------------------+------------------------------------+ -| FEDCLOUD_MYTOKEN_SERVER | --mytoken-server | https://mytoken.data.kit.edu | -+-----------------------------+---------------------------------+------------------------------------+ -| FEDCLOUD_MYTOKEN | --mytoken | | -+-----------------------------+---------------------------------+------------------------------------+ -| OPENSTACK_AUTH_PROTOCOL | --openstack-auth-protocol | openid | -+-----------------------------+---------------------------------+------------------------------------+ -| OPENSTACK_AUTH_PROVIDER | --openstack-auth-provider | egi.eu | -+-----------------------------+---------------------------------+------------------------------------+ -| OPENSTACK_AUTH_TYPE | --openstack-auth-type | v3oidcaccesstoken | -+-----------------------------+---------------------------------+------------------------------------+ -| EGI_VO | --vo | | -+-----------------------------+---------------------------------+------------------------------------+ - -For convenience, always set the frequently used options like tokens via environment variables, that can save a lot of -time. - -Shell completion -**************** - -Shell completion for *fedcloud* command in *bash* can be activated by executing the following command: - -:: - - $ eval "$(_FEDCLOUD_COMPLETE=bash_source fedcloud)" - -The command above may affect responsiveness of the shell. For long work, it is recommended to copy the -`fedcloud_bash_completion.sh script -`_ to a local file, and -source it from ~/.bashrc. Refer `Click documentation -`_ for a long explanation. - -After enabling shell completion, press twice for shell completion: - -:: - - $ fedcloud site - env list save-config show show-project-id - - -fedcloud --help command -*********************** - -* **"fedcloud --help"** command will print help message. When using it in combination with other - commands, e.g. **"fedcloud token --help"**, **"fedcloud token check --hep"**, it will print list of options for the - corresponding commands - -:: - - $ fedcloud --help - Usage: fedcloud [OPTIONS] COMMAND [ARGS]... - - Options: - --help Show this message and exit. - - Commands: - endpoint Endpoint command group for interaction with GOCDB and endpoints - openstack Executing OpenStack commands on site and VO - openstack-int Interactive OpenStack client on site and VO - site Site command group for manipulation with site configurations - token Token command group for manipulation with tokens - - -fedcloud token commands -*********************** - -* **"fedcloud token check --oidc-access-token "**: Check the expiration time of access token, so users can know whether - they need to refresh it. As mentioned before, access token may be given via environment variable *OIDC_ACCESS_TOKEN*, - so the option *--oidc-access-token* is not shown in all examples bellows, even if the option is required. - -:: - - $ fedcloud token check - Access token is valid to 2021-01-02 01:25:39 UTC - Access token expires in 3571 seconds - - -* **"fedcloud token list-vos --oidc-access-token "** : Print the list of VO memberships according to EGI Check-in - -:: - - $ fedcloud token list-vos - eosc-synergy.eu - fedcloud.egi.eu - training.egi.eu - - -fedcloud endpoint commands -************************** - -**"fedcloud endpoint"** commands are complementary part of the **"fedcloud site"** commands. Instead of using site -configurations defined in files saved in GitHub repository or local disk, the commands try to get site information -directly from GOCDB (Grid Operations Configuration Management Database) https://goc.egi.eu/ or make probe test on sites - -* **"fedcloud endpoint list"** : List of endpoints of sites defined in GOCDB. - -:: - - $ fedcloud endpoint list - Site type URL - ------------------ ------------------ ------------------------------------------------ - IFCA-LCG2 org.openstack.nova https://api.cloud.ifca.es:5000/v3/ - IN2P3-IRES org.openstack.nova https://sbgcloud.in2p3.fr:5000/v3 - ... - - -* **"fedcloud endpoint projects --site --oidc-access-token "** : List of projects to which the owner - of the access token has access at the given site - -:: - - $ fedcloud endpoint projects --site IFCA-LCG2 - id Name enabled site - -------------------------------- -------------------------- --------- --------- - 2a7e2cd4b6dc4e609dd934964c1715c6 VO:demo.fedcloud.egi.eu True IFCA-LCG2 - 3b9754ad8c6046b4aec43ec21abe7d8c VO:eosc-synergy.eu True IFCA-LCG2 - ... - -If the site is set to *ALL_SITES*, or the argument *-a* is used, the command will show accessible projects from all sites of the EGI Federated Cloud. - - -* **"fedcloud endpoint vos --site --oidc-access-token "** : List of Virtual Organisations (VOs) - to which the owner of the access token has access at the given site - -:: - - $ fedcloud endpoint vos --site IFCA-LCG2 - VO id Project name enabled site - ---------------- -------------------------------- ------------------- --------- --------- - vo.access.egi.eu 233f045cb1ff46842a15ebb33af69460 VO:vo.access.egi.eu True IFCA-LCG2 - training.egi.eu d340308880134d04294097524eace710 VO:training.egi.eu True IFCA-LCG2 - ... - -If the site is set to *ALL_SITES*, or the argument *-a* is used, the command will show accessible VOs from all sites of the EGI Federated Cloud. - -:: - - $ fedcloud endpoint vos -a - VO id Project name enabled site - ------------------- -------------------------------- ------------------- --------- ----------------- - vo.access.egi.eu 233f045cb1ff46842a15ebb33af69460 VO:vo.access.egi.eu True IFCA-LCG2 - training.egi.eu d340308880134d04294097524eace710 VO:training.egi.eu True IFCA-LCG2 - vo.access.egi.eu 7101022b9ae74ed9ac1a574497279499 EGI_access True IN2P3-IRES - vo.access.egi.eu 5bbdb5c1e0b2bcbac29904f4ac22dcaa vo_access_egi_eu True UNIV-LILLE - vo.access.egi.eu 4cab325ca8c2495bf2d4e8f230bcd51a VO:vo.access.egi.eu True INFN-PADOVA-STACK - ... - - -* **"fedcloud endpoint token --site --project-id --oidc-access-token "** : Get - OpenStack keystone scoped token on the site for the project ID. - -:: - - $ fedcloud endpoint token --site IFCA-LCG2 --project-id 3b9754ad8c6046b4aec43ec21abe7d8c - export OS_TOKEN="gAAAAA..." - - -* **"fedcloud endpoint env --site --project-id --oidc-access-token "** : Print - environment variables for working with the project ID on the site. - -:: - - $ fedcloud endpoint env --site IFCA-LCG2 --project-id 3b9754ad8c6046b4aec43ec21abe7d8c - # environment for IFCA-LCG2 - export OS_AUTH_URL="https://api.cloud.ifca.es:5000/v3/" - export OS_AUTH_TYPE="v3oidcaccesstoken" - export OS_IDENTITY_PROVIDER="egi.eu" - export OS_PROTOCOL="openid" - export OS_ACCESS_TOKEN="..." - - -fedcloud ec3 commands -************************** - -**"fedcloud ec3"** commands are helper commands for deploying EC3 (Elastic Cloud Compute Cluster) in Cloud -via Infrastructure Manager. The commands will create necessary template and authorization files for EC3 client. - -* **"fedcloud ec3 init --site --vo --oidc-access-token --auth-file auth.dat --template-dir - ./templates"** : Generate authorization file (by default *auth.dat*) and template file (by default - *./templates/refresh.radl*) for EC3 client. - -:: - - $ fedcloud ec3 init --site CESGA --vo vo.access.egi.eu - - -* **"fedcloud ec3 refresh --site --vo --oidc-access-token --auth-file auth.dat"** : - Refresh the access token stored in authorization file (by default *auth.dat*). - -:: - - $ fedcloud ec3 init --site CESGA --vo vo.access.egi.eu - - - -fedcloud site commands -********************** - -**"fedcloud site"** commands will read site configurations and manipulate with them. If the local site configurations -exist at *~/.config/fedcloud/site-config/*, **fedcloud** will read them from there, otherwise the commands will read -from `GitHub repository `_. - -By default, **fedcloud** does not save anything on local disk, users have to save the site configuration to local disk -explicitly via **"fedcloud site save-config"** command. The advantage of having local -site configurations, beside faster loading, is to give users ability to make customizations, e.g. add additional VOs, -remove sites they do not have access, and so on. - -* **"fedcloud site save-config"** : Read the default site configurations from GitHub - and save them to *~/.config/fedcloud/site-config/* local directory. The command will overwrite existing site configurations - in the local directory. - -:: - - $ fedcloud site save-config - Saving site configs to directory /home/viet/.config/fedcloud/site-config/ - - -After saving site configurations, users can edit and customize them, e.g. remove inaccessible sites, add new -VOs and so on. - -* **"fedcloud site list"** : List of existing sites in the site configurations - -:: - - $ fedcloud site list - 100IT - BIFI - CESGA - ... - - -* **"fedcloud site list --vo "** : List all sites supporting a Virtual Organization - -:: - - $ fedcloud site vo-list --vo vo.access.egi.eu - BIFI - CENI - CESGA-CLOUD - ... - - -* **"fedcloud site show --site "** : Show configuration of the corresponding site. - -:: - - $ fedcloud site show --site IISAS-FedCloud - endpoint: https://cloud.ui.savba.sk:5000/v3/ - gocdb: IISAS-FedCloud - vos: - - auth: - project_id: a22bbffb007745b2934bf308b0a4d186 - name: covid19.eosc-synergy.eu - - auth: - project_id: 51f736d36ce34b9ebdf196cfcabd24ee - name: eosc-synergy.eu - - -* **"fedcloud site show-project-id --site --vo "**: show the project ID of the VO on the site. - -:: - - $ fedcloud site show-project-id --site IISAS-FedCloud --vo eosc-synergy.eu - export OS_AUTH_URL="https://cloud.ui.savba.sk:5000/v3/" - export OS_PROJECT_ID="51f736d36ce34b9ebdf196cfcabd24ee" - - -* **"fedcloud site env --site --vo "**: set OpenStack environment variable for the VO on the site. - -:: - - $ fedcloud site env --site IISAS-FedCloud --vo eosc-synergy.eu - export OS_AUTH_URL="https://cloud.ui.savba.sk:5000/v3/" - export OS_AUTH_TYPE="v3oidcaccesstoken" - export OS_IDENTITY_PROVIDER="egi.eu" - export OS_PROTOCOL="openid" - export OS_PROJECT_ID="51f736d36ce34b9ebdf196cfcabd24ee" - # Remember to set OS_ACCESS_TOKEN, e.g. : - # export OS_ACCESS_TOKEN=`oidc-token egi` - - -The main differences between *"fedcloud endpoint env"* and *"fedcloud site env"* commands are that the second command -needs VO name as input parameter instead of project ID. The command may set also environment variable OS_ACCESS_TOKEN, -if access token is provided, otherwise it will print notification. - - -fedcloud select commands -*************************** - -* **"fedcloud select flavor --site --vo --oidc-access-token --flavor-specs "** : - Select flavor according to the specification in *flavor-specs*. The specifications may be repeated, - e.g. *--flavor-specs "VCPUs==2" --flavor-specs "RAM>=2048"*, or may be joined, e.g. - *--flavor-specs "VCPUs==2 & Disk>10"*. For frequently used specs, short-option alternatives are available, e.g. - *--vcpus 2* is equivalent to *--flavor-specs "VCPUs==2"*. The output is sorted, flavors using less resources - (in the order: GPUs, CPUs, RAM, Disk) are placed on the first places. Users can choose to print only the best-matched - flavor with *--output-format first* (suitable for scripting) or the full list of all matched flavors in list/YAML/JSON - format. - -:: - - $ fedcloud select flavor --site IISAS-FedCloud --vo vo.access.egi.eu --flavor-specs "RAM>=2096" --flavor-specs "Disk > 10" --output-format list - m1.medium - m1.large - m1.xlarge - m1.huge - g1.c08r30-K20m - g1.c16r60-2xK20m - - -* **"fedcloud select image --site --vo --oidc-access-token --image-specs "** : - Select image according to the specification in *image-specs*. The specifications may be repeated, - e.g. *--image-specs "Name=~Ubuntu" --image-specs "Name=~'20.04'"*. The output is sorted, newest images - are placed on the first places. Users can choose to print only the best-matched - image with *--output-format first* (suitable for scripting) or the full list of all matched images in list/YAML/JSON - format. - -:: - - $ fedcloud select image --site INFN-CATANIA-STACK --vo training.egi.eu --image-specs "Name =~ Ubuntu" --output-format list - TRAINING.EGI.EU Image for EGI Docker [Ubuntu/18.04/VirtualBox] - TRAINING.EGI.EU Image for EGI Ubuntu 20.04 [Ubuntu/20.04/VirtualBox] - - -* **"fedcloud select network --site --vo --oidc-access-token --network-specs "** : - Select network according to the specification in *network-specs*. User can choose to select only public or private - network, or both (default). The output is sorted in the order: public, shared, - private. Users can choose to print only the best-matched network with *--output-format first* - (suitable for scripting) or the full list of all matched networks in list/YAML/JSON format. - -:: - - $ fedcloud select network --site IISAS-FedCloud --vo training.egi.eu --network-specs default --output-format list - public-network - private-network - - -fedcloud openstack commands -*************************** - -* **"fedcloud openstack --site --vo --oidc-access-token "** : Execute an - OpenStack command on the site and VO. Examples of OpenStack commands are *"image list"*, *"server list"* and can be used - with additional options for the commands, e.g. *"image list --long"*, *"server list --format json"*. The list of all - OpenStack commands, and their parameters/usages are available - `here `_. - -:: - - $ fedcloud openstack image list --site IISAS-FedCloud --vo eosc-synergy.eu - Site: IISAS-FedCloud, VO: eosc-synergy.eu - +--------------------------------------+-------------------------------------------------+--------+ - | ID | Name | Status | - +--------------------------------------+-------------------------------------------------+--------+ - | 862d4ede-6a11-4227-8388-c94141a5dace | Image for EGI CentOS 7 [CentOS/7/VirtualBox] | active | - ... - - -If the site is *ALL_SITES*, the OpenStack command will be executed on all sites in EGI Federated Cloud. - -* **"fedcloud openstack-int --site --vo --oidc-access-token "** : Call OpenStack client without - command, so users can work with OpenStack site in interactive mode. This is useful when users need to perform multiple - commands successively. For example, users may need get list of images, list of flavors, list of networks before - creating a VM. OIDC authentication is done only once at the beginning, then the keystone token is cached and will - be used for successive commands without authentication via CheckIn again. - -:: - - $ fedcloud openstack-int --site IISAS-FedCloud --vo eosc-synergy.eu - (openstack) image list - +--------------------------------------+-------------------------------------------------+--------+ - | ID | Name | Status | - +--------------------------------------+-------------------------------------------------+--------+ - | 862d4ede-6a11-4227-8388-c94141a5dace | Image for EGI CentOS 7 [CentOS/7/VirtualBox] | active | - ... - (openstack) flavor list - +--------------------------------------+-----------+-------+------+-----------+-------+-----------+ - | ID | Name | RAM | Disk | Ephemeral | VCPUs | Is Public | - +--------------------------------------+-----------+-------+------+-----------+-------+-----------+ - | 5bd8397c-b97f-462d-9d2b-5b533844996c | m1.small | 2048 | 10 | 0 | 1 | True | - | df25f80f-ed19-4e0b-805e-d34620ba0334 | m1.medium | 4096 | 40 | 0 | 2 | True | - ... - (openstack) - -fedcloud secret commands -*************************** - -The **"fedcloud secret"** commands are described in details in the documentation of the -`Secret management service `_. diff --git a/fedcloudclient/auth.py b/fedcloudclient/auth.py new file mode 100644 index 0000000..b645c12 --- /dev/null +++ b/fedcloudclient/auth.py @@ -0,0 +1,269 @@ +""" +Class for managing tokens +""" + +import re +import time +from datetime import datetime +import jwt +import liboidcagent as agent +import requests + +from fedcloudclient.conf import CONF +from fedcloudclient.exception import TokenError +from fedcloudclient.logger import log_and_raise +from fedcloudclient.conf import save_config, DEFAULT_CONFIG_LOCATION + +# pylint: disable=too-few-public-methods +class Token: + """ + Abstract object for managing tokens + + """ + def __init__(self): + pass + +# pylint: disable=too-few-public-methods +class OIDCToken(Token): + """ + OIDC tokens. Managing access tokens, oidc-agent account and mytoken + + """ + + def __init__(self, access_token=None): + super().__init__() + self.access_token = access_token + self.payload = None + self.oidc_agent_account = None + self.mytoken = None + self.user_id = None + self._vo_pattern = "urn:mace:egi.eu:group:(.+?):(.+:)*role=member#aai.egi.eu" + self.request_json=None + self._min_access_token_time=CONF["_MIN_ACCESS_TOKEN_TIME"] + self.conf=CONF + if access_token is not None: + self.decode_token() + self.oidc_discover() + + def decode_token(self) -> dict: + """ + Decoding access token to payload + :return: + + """ + if not self.payload: + try: + self.payload = jwt.decode(self.access_token, options={"verify_signature": False}) + self.user_id = self.payload["sub"] + except jwt.exceptions.InvalidTokenError: + error_msg = "Invalid access token" + log_and_raise(error_msg, TokenError) + + return self.payload + + def get_checkin_id(self,access_token): + """ + Get EGI Check-in ID from access token + + :param oidc_token: the token + + :return: Check-in ID + + """ + self.access_token=access_token + payload = self.decode_token() + if payload is None: + return None + return payload["sub"] + + def get_user_id(self) -> str: + """ + Return use ID + :return: + + """ + + if not self.payload: + self.decode_token() + return None + return self.user_id + + def get_token_from_oidc_agent(self, oidc_agent_account: str) -> str: + """ + Get access token from oidc-agent + :param oidc_agent_account: account name in oidc-agent + :return: access token, and set internal token, raise TokenError on None + """ + + if oidc_agent_account: + try: + access_token = agent.get_access_token( + oidc_agent_account, + min_valid_period=CONF.get("_MIN_ACCESS_TOKEN_TIME"), + application_hint="fedcloudclient", + ) + self.access_token = access_token + self.oidc_agent_account = oidc_agent_account + self.conf["oidc_agent_account"]=str(oidc_agent_account) + save_config(DEFAULT_CONFIG_LOCATION,self.conf) + return access_token + + except agent.OidcAgentError as exception: + error_msg = f"Error getting access token from oidc-agent in -def get_token_from_oidc_agent()-: {exception}" + log_and_raise(error_msg, TokenError) + return None + else: + error_msg = f"Error getting access token from oidc-agent: {oidc_agent_account}" + log_and_raise(error_msg, TokenError) + return None + + def get_token_from_mytoken(self, mytoken: str, mytoken_server: str = None) -> str: + """ + Get access token from mytoken server + :param mytoken: + :param mytoken_server: + :return: access token, or None on error + + """ + if not mytoken_server: + mytoken_server = CONF.get("mytoken_server") + + if mytoken: + try: + data = { + "grant_type": "mytoken", + "mytoken": mytoken, + } + try: + req = requests.post( + mytoken_server + "/api/v0/token/access", + json=data,) + + self.conf["mytoken"]=str(mytoken) + save_config(DEFAULT_CONFIG_LOCATION,self.conf) + + except requests.exceptions.Timeout as err: + error_msg = f"Timeout for requests in mytoken: {err}" + log_and_raise(error_msg, err) + return None + req.raise_for_status() + access_token = req.json().get("access_token") + self.access_token = access_token + self.mytoken = mytoken + return access_token + + except requests.exceptions.HTTPError as exception: + error_msg = f"Error getting access token from mytoken server: {exception}" + log_and_raise(error_msg, TokenError) + return None + else: + error_msg = f"Error getting access token from mytoken server: mytoken is {mytoken}" + log_and_raise(error_msg, TokenError) + return None + + def multiple_token(self, access_token: str, oidc_agent_account: str, mytoken: str, mytoken_server = None) -> str: + """ + Select valid token from multiple options + :param access_token: + :param oidc_agent_account: + :param mytoken: + :return: + + """ + if mytoken: + try: + access_token=self.get_token_from_mytoken(mytoken) + return access_token + except TokenError: + pass + if oidc_agent_account: + try: + access_token=self.get_token_from_oidc_agent(oidc_agent_account) + return access_token + except TokenError: + pass + if mytoken_server: + pass + + if access_token: + self.access_token = access_token + return access_token + log_and_raise("Cannot get access token", TokenError) + return None + + def oidc_discover(self) -> dict: + """ + :param oidc_url: CheckIn URL get from payload + :return: JSON object of OIDC configuration + + """ + oidc_url=self.payload["iss"] + request = requests.get(oidc_url + "/.well-known/openid-configuration") + request.raise_for_status() + self.request_json=request.json() + return self.request_json + + def check_token(self, access_token, verbose=False): + """ + Check validity of access token + + :param verbose: + :param oidc_token: the token to check + :return: access token, or None on error + + """ + self.access_token=access_token + payload = self.decode_token() + if payload is None: + return None + + exp_timestamp = int(payload["exp"]) + current_timestamp = int(time.time()) + exp_time_in_sec = exp_timestamp - current_timestamp + + if exp_time_in_sec < self._min_access_token_time: + error_msg=f"Error: Expired access token in {exp_time_in_sec}" + log_and_raise(error_msg,TokenError) + return None + + if verbose: + exp_time_str = datetime.utcfromtimestamp(exp_timestamp).strftime( + "%Y-%m-%d %H:%M:%S" + ) + print(f"Token is valid until {exp_time_str} UTC") + if exp_time_in_sec < 24 * 3600: + print(f"Token expires in {exp_time_in_sec} seconds") + else: + exp_time_in_days = exp_time_in_sec // (24 * 3600) + print(f"Token expires in {exp_time_in_days} days") + + return access_token + + def token_list_vos(self,access_token): + """ + List VO memberships in EGI Check-in + :return: list of VO names + + """ + self.access_token=access_token + oidc_ep = self.request_json + try: + request = requests.get( + oidc_ep["userinfo_endpoint"], + headers={"Authorization": f"Bearer {self.access_token}"}) + + except requests.exceptions.Timeout as err: + error_msg = f"Timeout for requests in list-vos: {err}" + log_and_raise(error_msg, err) + return None + + request.raise_for_status() + vos = set() + pattern = re.compile(self._vo_pattern) + for claim in request.json().get("eduperson_entitlement", []): + vo = pattern.match(claim) # pylint: disable=invalid-name + if vo: + vos.add(vo.groups()[0]) + request.raise_for_status() + + return sorted(vos) diff --git a/fedcloudclient/auth_test.py b/fedcloudclient/auth_test.py new file mode 100644 index 0000000..44db551 --- /dev/null +++ b/fedcloudclient/auth_test.py @@ -0,0 +1,102 @@ +""" +Testing unit for auth.py +""" +import os + +from fedcloudclient.auth import OIDCToken +from fedcloudclient.logger import log_and_raise +from fedcloudclient.exception import TokenError + +def verify_mytoken(mytoken: str) -> str: + """ + Get access token from mytoken, decode them, get user ID and verify + """ + token = OIDCToken() + try: + access_token_mytoken=token.get_token_from_mytoken(mytoken, None) + return access_token_mytoken + except TokenError: + err_msg="No MYTOKEN" + return log_and_raise(err_msg,TokenError) + + +def verify_oidc_agent(user_id: str) -> str: + """ Verify token access from oidc-agent""" + token = OIDCToken() + try: + access_token_oidc=token.get_token_from_oidc_agent(user_id) + return access_token_oidc + except TokenError: + err_msg="No FEDCLOUD_OIDC_AGENT_ACCOUNTT" + return log_and_raise(err_msg,TokenError) + + +def verify_access_token(access_token:str) -> str: + """ Verify access_token """ + token = OIDCToken() + try: + token.access_token=access_token + return token.access_token + except TokenError: + err_msg="Not valid ACCESS_TOKEN" + return log_and_raise(err_msg,TokenError) + +def verify_user_id(access_token:str) -> str: + """ Check user id from access_token """ + token = OIDCToken() + token.access_token=access_token + try: + user_id=token.get_user_id() + return user_id + except TokenError: + err_msg="No user ID from access_token" + return log_and_raise(err_msg,TokenError) + +def verify_pyload(access_token:str) -> dict: + """ Get payload, request_json, list_vos from access_token """ + token = OIDCToken() + token.access_token=access_token + try: + token.get_user_id() + payload=token.payload + request_json=token.oidc_discover() + list_vos=token.token_list_vos(access_token) + return payload,request_json,list_vos + except TokenError: + err_msg="Not valid ACCESS_TOKEN" + return log_and_raise(err_msg,TokenError) + + + +def printing_dict(var_dict: dict) -> None: + """ Printing dictionary """ + for item in var_dict: + print(f"{item}:\t {var_dict[item]}") + + +if __name__ == "__main__": + print("Start of verifying auth.py") + + access_token1= os.environ.get("FEDCLOUD_ACCESS_TOKEN","") + #access_token_check=verify_access_token(access_token1) + + #payload1,request_json1,list_vos1=verify_pyload(access_token_check) + + mytoken1=os.environ.get("FEDCLOUD_MYTOKEN","") + #access_token_mytok=verify_mytoken(mytoken1) + + oidc_agent_name=os.environ.get("FEDCLOUD_OIDC_AGENT_ACCOUNT","") + print(f"OIDC_AGENT name: {oidc_agent_name}") + access_token_oidc1=verify_oidc_agent(oidc_agent_name) + + user_id1=verify_user_id(access_token_oidc1) + payload1,request_json1,list_vos1=verify_pyload(access_token_oidc1) + + + print(f"{type(payload1)}") + printing_dict(payload1) + print("-------------------------------------------------") + printing_dict(request_json1) + print("-------------------------------------------------") + print(list_vos1) + print("Break") diff --git a/fedcloudclient/checkin.py b/fedcloudclient/checkin.py index 41b03bd..9071643 100644 --- a/fedcloudclient/checkin.py +++ b/fedcloudclient/checkin.py @@ -2,238 +2,10 @@ Implementation of "fedcloud token" commands for interactions with EGI Check-in and access tokens """ -import re -import sys -import time -from datetime import datetime import click -import jwt -import liboidcagent as agent -import requests - -from fedcloudclient.decorators import ( - oidc_params, -) - -# Minimal lifetime of the access token is 30s and max 24h -_MIN_ACCESS_TOKEN_TIME = 30 - -VO_PATTERN = "urn:mace:egi.eu:group:(.+?):(.+:)*role=member#aai.egi.eu" - - -def print_error(message, quiet): - """ - Print error message to stderr if not quiet - """ - if not quiet: - print(message, file=sys.stderr) - - -def decode_token(oidc_access_token): - """ - Decoding access token to a dict - :param oidc_access_token: - :return: dict with token info - """ - try: - payload = jwt.decode(oidc_access_token, options={"verify_signature": False}) - except jwt.exceptions.InvalidTokenError: - print_error("Error: Invalid access token.", False) - return None - return payload - - -def oidc_discover(oidc_url): - """ - Discover OIDC endpoints - - :param oidc_url: CheckIn URL - - :return: JSON object of OIDC configuration - """ - request = requests.get(oidc_url + "/.well-known/openid-configuration") - request.raise_for_status() - return request.json() - - -def get_token_from_oidc_agent(oidc_agent_account, quiet=False): - """ - Get access token from oidc-agent - :param quiet: - :param oidc_agent_account: account name in oidc-agent - :return: access token, or None on error - """ - - if oidc_agent_account: - try: - access_token = agent.get_access_token( - oidc_agent_account, - min_valid_period=_MIN_ACCESS_TOKEN_TIME, - application_hint="fedcloudclient", - ) - return access_token - except agent.OidcAgentError as exception: - print_error( - "Error getting access token from oidc-agent\n" - f"Error message: {exception}", - quiet, - ) - return None - - -def get_token_from_mytoken_server(mytoken, mytoken_server, quiet=False): - """ - Get access token from mytoken server - :param quiet: - :param mytoken: - :param mytoken_server: - :return: access token, or None on error - """ - - if mytoken: - try: - data = { - "grant_type": "mytoken", - "mytoken": mytoken, - } - req = requests.post( - mytoken_server + "/api/v0/token/access", - json=data, - ) - req.raise_for_status() - return req.json().get("access_token") - except requests.exceptions.HTTPError as exception: - print_error( - "Error getting access token from mytoken\n" - f"Error message: {exception}", - quiet, - ) - return None - - -def check_token(oidc_token, verbose=False): - """ - Check validity of access token - - :param verbose: - :param oidc_token: the token to check - :return: access token, or None on error - """ - - payload = decode_token(oidc_token) - if payload is None: - return None - - exp_timestamp = int(payload["exp"]) - current_timestamp = int(time.time()) - exp_time_in_sec = exp_timestamp - current_timestamp - - if exp_time_in_sec < _MIN_ACCESS_TOKEN_TIME: - print_error("Error: Expired access token.", False) - return None - - if verbose: - exp_time_str = datetime.utcfromtimestamp(exp_timestamp).strftime( - "%Y-%m-%d %H:%M:%S" - ) - print(f"Token is valid until {exp_time_str} UTC") - if exp_time_in_sec < 24 * 3600: - print(f"Token expires in {exp_time_in_sec} seconds") - else: - exp_time_in_days = exp_time_in_sec // (24 * 3600) - print(f"Token expires in {exp_time_in_days} days") - - return oidc_token - - -def get_checkin_id( - oidc_token, -): - """ - Get EGI Check-in ID from access token - - :param oidc_token: the token - - :return: Check-in ID - """ - payload = decode_token(oidc_token) - if payload is None: - return None - return payload["sub"] - - -def get_access_token( - oidc_access_token, - oidc_agent_account, - mytoken, - mytoken_server, -): - """ - Get access token - Generates new access token from oidc-agent - or mytoken - - Check expiration time of access token - Raise error if no valid token exists - - :param oidc_access_token: - :param oidc_agent_account: - :param mytoken: - :param mytoken_server: - :return: access token - """ - - access_token = None - - # access token via parameter has the highest priority - if oidc_access_token: - access_token = check_token(oidc_access_token) - - # then try to get access token from mytoken server - if mytoken and access_token is None: - access_token = get_token_from_mytoken_server( - mytoken, mytoken_server, quiet=False - ) - - # then, try to get access token from oidc-agent - if oidc_agent_account and access_token is None: - access_token = get_token_from_oidc_agent(oidc_agent_account, quiet=False) - - if access_token is None: - # Nothing available - raise SystemExit( - "Error: An access token is needed for the operation. You can specify " - "access token directly via --oidc-access-token option or use oidc-agent " - "via --oidc-agent-account or mytoken via --mytoken" - ) - - return access_token - - -def token_list_vos(oidc_access_token): - """ - List VO memberships in EGI Check-in - - :param oidc_access_token: - - :return: list of VO names - """ - oidc_url = decode_token(oidc_access_token)["iss"] - oidc_ep = oidc_discover(oidc_url) - request = requests.get( - oidc_ep["userinfo_endpoint"], - headers={"Authorization": f"Bearer {oidc_access_token}"}, - ) - - request.raise_for_status() - vos = set() - pattern = re.compile(VO_PATTERN) - for claim in request.json().get("eduperson_entitlement", []): - vo = pattern.match(claim) - if vo: - vos.add(vo.groups()[0]) - return sorted(vos) +from fedcloudclient.auth import OIDCToken +from fedcloudclient.decorators import oidc_params @click.group() @@ -242,14 +14,14 @@ def token(): Get details of access token """ - @token.command() @oidc_params def check(access_token): """ Check validity of access token """ - check_token(access_token, verbose=True) + token_check=OIDCToken() + token_check.check_token(access_token, verbose=True) @token.command() @@ -258,7 +30,8 @@ def list_vos(access_token): """ List VO membership(s) of access token """ - vos = token_list_vos(access_token) + token_vos=OIDCToken(access_token) + vos = token_vos.token_list_vos(access_token) print("\n".join(vos)) diff --git a/fedcloudclient/cli.py b/fedcloudclient/cli.py index 252f02a..2eb18e0 100644 --- a/fedcloudclient/cli.py +++ b/fedcloudclient/cli.py @@ -5,7 +5,7 @@ import click from fedcloudclient.checkin import token -from fedcloudclient.ec3 import ec3 +from fedcloudclient.conf import config from fedcloudclient.endpoint import endpoint from fedcloudclient.openstack import openstack, openstack_int from fedcloudclient.secret import secret @@ -23,13 +23,12 @@ def cli(): cli.add_command(token) cli.add_command(endpoint) -cli.add_command(ec3) cli.add_command(site) cli.add_command(secret) cli.add_command(select) cli.add_command(openstack) cli.add_command(openstack_int) - +cli.add_command(config) if __name__ == "__main__": cli() diff --git a/fedcloudclient/conf.py b/fedcloudclient/conf.py new file mode 100644 index 0000000..75f9372 --- /dev/null +++ b/fedcloudclient/conf.py @@ -0,0 +1,185 @@ +""" +Read/write configuration files +""" +import json +import os +import sys +from pathlib import Path +import textwrap + +import click +import yaml +from tabulate import tabulate +from fedcloudclient.exception import ConfigError + + + +DEFAULT_CONFIG_LOCATION = Path.home() / ".config/fedcloud/config.yaml" +DEFAULT_SETTINGS = { + "site": "IISAS-FedCloud", + "vo": "vo.access.egi.eu", + "site_list_url": "https://raw.githubusercontent.com/tdviet/fedcloudclient/master/config/sites.yaml", + "site_dir": str(Path.home() / ".config/fedcloud/site-config/"), + "oidc_url": "https://aai.egi.eu/auth/realms/egi", + "gocdb_public_url": "https://goc.egi.eu/gocdbpi/public/", + "gocdb_service_group": "org.openstack.nova", + "vault_endpoint": "https://vault.services.fedcloud.eu:8200", + "vault_role": "", + "vault_mount_point": "/secrets/", + "vault_locker_mount_point": "/v1/cubbyhole/", + "vault_salt": "fedcloud_salt", + "log_file": str(Path.home() / ".config/fedcloud/logs/fedcloud.log"), + "log_level": "DEBUG", + "log_config_file": str(Path.home() / ".config/fedcloud/logging.conf"), + "requests_cert_file": str(Path.home() / ".config/fedcloud/cert/certs.pem"), + "oidc_agent_account": "egi", + "min_access_token_time": 30, + "mytoken_server": "https://mytoken.data.kit.edu", + "os_protocol": "openid", + "os_auth_type": "v3oidcaccesstoken", + "os_identity_provider": "egi.eu", + "_MIN_ACCESS_TOKEN_TIME": 30 +} + +def init_default_config(): + """ + Initialisation of default settings + + :return dictionary of + """ + default_config_init=DEFAULT_SETTINGS + return default_config_init + +def save_config(filename: str, config_data: dict): + """ + Save configuration to file + + :param filename: name of config file + :param config_data: dict containing configuration + :return: None + """ + config_file = Path(filename).resolve() + try: + with config_file.open(mode="w+", encoding="utf-8") as file: + yaml.dump(config_data, file) + except yaml.YAMLError as exception: + error_msg = f"Error during saving configuration to {filename}: {exception}" + raise ConfigError(error_msg) from exception + + +def load_config(filename: str) -> dict: + """ + Load configuration file + :param filename: + :return: configuration data + """ + + config_file = Path(filename).resolve() + + if config_file.is_file(): + try: + with config_file.open(mode="r", encoding="utf-8") as file: + return yaml.safe_load(file) + except yaml.YAMLError as exception: + error_msg = f"Error during loading configuration from {filename}: {exception}" + raise ConfigError(error_msg) from exception + else: + return {} + + +def load_env() -> dict: + """ + Load configs from environment variables + :return: config + """ + env_config = {} + for env in os.environ: + if env.startswith("FEDCLOUD_"): + config_key = env[9:].lower() + env_config[config_key] = os.environ[env] + return env_config + + +def init_config() -> dict: + """ + Init config moduls + :return: actual config + """ + env_config = load_env() + config_file = env_config.get("config_file", DEFAULT_CONFIG_LOCATION) + + try: + saved_config = load_config(config_file) + except ConfigError: + saved_config = {} + + act_config = {**DEFAULT_SETTINGS, **env_config, **saved_config} + return act_config + +@click.group() +def config(): + """ + Managing fedcloud configurations + """ + +@config.command() +@click.option( + "--config-file", + type=click.Path(dir_okay=False), + default=DEFAULT_CONFIG_LOCATION, + help="configuration file", + envvar="FEDCLOUD_CONFIG_FILE", + show_default=True, +) +def create(config_file: str): + """Create default configuration file""" + save_config(config_file, CONF) + + +@config.command() +@click.option( + "--config-file", + type=click.Path(dir_okay=False), + default=DEFAULT_CONFIG_LOCATION, + help="configuration file", + envvar="FEDCLOUD_CONFIG_FILE", + show_default=True, +) + +@click.option( + "--output-format", + "-f", + required=False, + help="Output format", + type=click.Choice(["text", "YAML", "JSON"], case_sensitive=False), +) + +@click.option( + "--source", "-s", + required=False, + help="Source of configuration data", + type=click.Choice(["DEFAULT_SETTINGS", "env_config", "saved_config"], case_sensitive=False), +) + +def show(config_file: str, output_format: str, source: str): + """Show config for FEDCLOUDCLIENT """ + saved_config = load_config(config_file) + env_config = load_env() + default_settings=init_default_config() + if source is not None: + act_config = vars()[source] + else: + act_config = {**default_settings, **env_config, **saved_config} + if output_format == "YAML": + yaml.dump(act_config, sys.stdout, sort_keys=False) + elif output_format == "JSON": + json.dump(act_config, sys.stdout, indent=4) + else: + wrapped_data = [ + ["\n".join(textwrap.wrap(cell, width=200)) if isinstance(cell, str) else cell for cell in row] + for row in act_config.items()] + + #print(tabulate(act_config.items(), headers=["parameter", "value"])) + print(tabulate(wrapped_data, headers=["parameter", "value"])) + +CONF = init_config() diff --git a/fedcloudclient/conf_test.py b/fedcloudclient/conf_test.py new file mode 100644 index 0000000..4b0c67f --- /dev/null +++ b/fedcloudclient/conf_test.py @@ -0,0 +1,57 @@ +""" +Testing unit for auth.py +""" + +import os + +from fedcloudclient.conf import CONF + + +def save_load_compare(): + """ + Save config to a temp file, load it and compare the result + """ + + config_data = { + "env1": "value1", + "env2": "value2", + "env3": "value3", + } + + config_file = "/tmp/test" + + CONF.save_config(config_file, config_data) + new_config = CONF.load_config(config_file) + + assert new_config == config_data + + +def load_env_merge_compare(): + """ + set OS env, load, and compare the result + """ + config_data = { + "env1": "value1", + "env2": "value2", + "env3": "value3", + } + + config_file = "/tmp/test" + + CONF.save_config(config_file, config_data) + saved_config = CONF.load_config(config_file) + + os.environ["FEDCLOUD_ENV1"] = "value10" + os.environ["FEDCLOUD_ENV4"] = "value4" + + env_config = CONF.load_env() + + act_config = {**config_data, **saved_config, **env_config} + + assert act_config["env1"] == "value10" + assert act_config["env4"] == "value4" + + +if __name__ == "__main__": + save_load_compare() + load_env_merge_compare() diff --git a/fedcloudclient/decorators.py b/fedcloudclient/decorators.py index 74eea23..2662381 100644 --- a/fedcloudclient/decorators.py +++ b/fedcloudclient/decorators.py @@ -1,19 +1,21 @@ """ Decorators for command-line parameters """ +import os from functools import wraps import click from click_option_group import ( RequiredAnyOptionGroup, - RequiredMutuallyExclusiveOptionGroup, - optgroup, + optgroup, MutuallyExclusiveOptionGroup, ) -DEFAULT_MYTOKEN_SERVER = "https://mytoken.data.kit.edu" -DEFAULT_PROTOCOL = "openid" -DEFAULT_AUTH_TYPE = "v3oidcaccesstoken" -DEFAULT_IDENTITY_PROVIDER = "egi.eu" +from fedcloudclient.conf import CONF +from fedcloudclient.exception import TokenError +from fedcloudclient.locker_auth import LockerToken +from fedcloudclient.vault_auth import VaultToken +from fedcloudclient.auth import OIDCToken +from fedcloudclient.logger import log_and_raise ALL_SITES_KEYWORDS = {"ALL_SITES", "ALL-SITES"} @@ -21,7 +23,7 @@ oidc_access_token_params = click.option( "--oidc-access-token", help="OIDC access token", - envvar="OIDC_ACCESS_TOKEN", + default=CONF.get("oidc_access_token"), metavar="token", ) @@ -29,8 +31,7 @@ site_params = click.option( "--site", help="Name of the site", - required=True, - envvar="EGI_SITE", + default=CONF.get("site"), metavar="site-name", ) @@ -51,13 +52,13 @@ def all_site_params(func): @optgroup.group( "Site", - cls=RequiredMutuallyExclusiveOptionGroup, + cls=MutuallyExclusiveOptionGroup, help="Single Openstack site or all sites", ) @optgroup.option( "--site", help="Name of the site or ALL_SITES", - envvar="EGI_SITE", + default=CONF.get("site"), metavar="site-name", ) @optgroup.option( @@ -77,8 +78,7 @@ def wrapper(*args, **kwargs): project_id_params = click.option( "--project-id", help="Project ID", - required=True, - envvar="OS_PROJECT_ID", + default=CONF.get("project_id"), metavar="project-id", ) @@ -95,8 +95,7 @@ def wrapper(*args, **kwargs): vo_params = click.option( "--vo", help="Name of the VO", - required=True, - envvar="EGI_VO", + default=CONF.get("vo"), metavar="vo-name", ) @@ -134,40 +133,39 @@ def oidc_params(func): @optgroup.option( "--oidc-agent-account", help="Account name in oidc-agent", - envvar="OIDC_AGENT_ACCOUNT", + default=CONF.get("oidc_agent_account"), metavar="account", ) @optgroup.option( "--oidc-access-token", help="OIDC access token", - envvar="OIDC_ACCESS_TOKEN", + default=os.getenv("FEDCLOUD_OIDC_ACCESS_TOKEN"), metavar="token", ) @optgroup.option( "--mytoken", help="Mytoken string", - envvar="FEDCLOUD_MYTOKEN", + default=CONF.get("mytoken"), metavar="mytoken", ) @optgroup.option( "--mytoken-server", help="Mytoken sever", - envvar="FEDCLOUD_MYTOKEN_SERVER", - default=DEFAULT_MYTOKEN_SERVER, - show_default=True, - metavar="mytoken-server", + default=CONF.get("mytoken_server"), + metavar="url", ) @wraps(func) def wrapper(*args, **kwargs): - from fedcloudclient.checkin import get_access_token - access_token = get_access_token( + token=OIDCToken() + access_token = token.multiple_token( kwargs.pop("oidc_access_token"), kwargs.pop("oidc_agent_account"), kwargs.pop("mytoken"), kwargs.pop("mytoken_server"), - ) + ) # pylint: disable=assignment-from-none kwargs["access_token"] = access_token + return func(*args, **kwargs) return wrapper @@ -183,26 +181,23 @@ def openstack_params(func): help="Only change default values if necessary", ) @optgroup.option( - "--openstack-auth-protocol", + "--os-protocol", help="Authentication protocol", - envvar="OPENSTACK_AUTH_PROTOCOL", - default=DEFAULT_PROTOCOL, + default=CONF.get("os_protocol"), show_default=True, metavar="", ) @optgroup.option( - "--openstack-auth-provider", + "--os-identity-provider", help="Identity provider", - envvar="OPENSTACK_AUTH_PROVIDER", - default=DEFAULT_IDENTITY_PROVIDER, + default=CONF.get("os_identity_provider"), show_default=True, metavar="", ) @optgroup.option( - "--openstack-auth-type", + "--os-auth-type", help="Authentication type", - envvar="OPENSTACK_AUTH_TYPE", - default=DEFAULT_AUTH_TYPE, + default=CONF.get("os_auth_type"), show_default=True, metavar="", ) @@ -363,58 +358,57 @@ def secret_token_params(func): @optgroup.option( "--locker-token", help="Locker token", - envvar="FEDCLOUD_LOCKER_TOKEN", + default=CONF.get("locker_token"), metavar="locker_token", ) + @optgroup.option( + "--vault-token", + help="Vault token", + default=CONF.get("vault_token"), + metavar="vault_token", + ) @optgroup.option( "--oidc-agent-account", help="Account name in oidc-agent", - envvar="OIDC_AGENT_ACCOUNT", + default=CONF.get("oidc_agent_account"), metavar="account", ) @optgroup.option( "--oidc-access-token", help="OIDC access token", - envvar="OIDC_ACCESS_TOKEN", + default=CONF.get("oidc_access_token"), metavar="token", ) @optgroup.option( "--mytoken", help="Mytoken string", - envvar="FEDCLOUD_MYTOKEN", + default=CONF.get("mytoken"), metavar="mytoken", ) - @optgroup.option( - "--mytoken-server", - help="Mytoken sever", - envvar="FEDCLOUD_MYTOKEN_SERVER", - default=DEFAULT_MYTOKEN_SERVER, - show_default=True, - metavar="mytoken-server", - ) @wraps(func) def wrapper(*args, **kwargs): - from fedcloudclient.checkin import get_access_token # If locker token is given, ignore OIDC token options locker_token = kwargs.pop("locker_token") + vault_token = kwargs.pop("vault_token") + access_token = kwargs.pop("oidc_access_token") + oidc_agent_account = kwargs.pop("oidc_agent_account") + mytoken = kwargs.pop("mytoken") + if locker_token: - kwargs.pop("oidc_access_token") - kwargs.pop("oidc_agent_account") - kwargs.pop("mytoken") - kwargs.pop("mytoken_server") - kwargs["access_token"] = None - kwargs["locker_token"] = locker_token - return func(*args, **kwargs) - - access_token = get_access_token( - kwargs.pop("oidc_access_token"), - kwargs.pop("oidc_agent_account"), - kwargs.pop("mytoken"), - kwargs.pop("mytoken_server"), - ) - kwargs["access_token"] = access_token - kwargs["locker_token"] = None + token = LockerToken(locker_token=locker_token) + elif vault_token: + token = VaultToken(vault_token=vault_token) + else: + token = VaultToken() + try: + token.multiple_token(access_token, oidc_agent_account, mytoken) + except TokenError: + error_msg=f"Can not access to the ACCESS_TOKEN: {TokenError}" + log_and_raise(error_msg, TokenError) + SystemExit(1) + + kwargs["token"] = token return func(*args, **kwargs) return wrapper diff --git a/fedcloudclient/ec3.py b/fedcloudclient/ec3.py deleted file mode 100644 index 09fe21b..0000000 --- a/fedcloudclient/ec3.py +++ /dev/null @@ -1,198 +0,0 @@ -""" -Implementation of ec3 commands for deploying EC3 (Elastic Cloud Computing -Cluster) in Cloud via Infrastructure Manager -""" - -import os -import time - -import click -import jwt - -from fedcloudclient.decorators import ( - ALL_SITES_KEYWORDS, - auth_file_params, - oidc_params, -) -from fedcloudclient.sites import find_endpoint_and_project_id, site_vo_params - -__MIN_EXPIRATION_TIME = 300 - -EC3_REFRESHTOKEN_TEMPLATE = """ -description refreshtoken ( - kind = 'component' and - short = 'Tool to refresh LToS access token.' and - content = 'Tool to refresh LToS access token.' -) -configure front ( -@begin - - vars: - CLIENT_ID: %(client_id)s - CLIENT_SECRET: %(client_secret)s - REFRESH_TOKEN: %(refresh_token)s - tasks: - - name: Check if docker is available - command: which docker - changed_when: false - failed_when: docker_installed.rc not in [0,1] - register: docker_installed - - name: local install of fedcloudclient - block: - - name: Create dir /usr/local/ec3/ - file: path=/usr/local/ec3/ state=directory - - name: install git - package: - name: git - state: present - - name: install fedcloudclient - pip: - name: - - fedcloudclient - - cron: - name: "refresh token" - minute: "*/5" - job: "[ -f /usr/local/ec3/auth.dat ] - && /usr/local/bin/fedcloudclient endpoint ec3-refresh - --oidc-client-id {{ CLIENT_ID }} - --oidc-client-secret {{ CLIENT_SECRET }} - --oidc-refresh-token {{ REFRESH_TOKEN }} - --auth-file /usr/local/ec3/auth.dat &> /var/log/refresh.log" - user: root - cron_file: refresh_token - state: present - when: docker_installed.rc not in [ 0 ] - - name: local install of fedcloudclient - block: - - cron: - name: "refresh token" - minute: "*/5" - job: "[ -f /usr/local/ec3/auth.dat ] - && docker run -v /usr/local/ec3/auth.dat:/usr/local/ec3/auth.dat - tdviet/fedcloudclient fedcloudclient endpoint ec3-refresh - --oidc-client-id {{ CLIENT_ID }} - --oidc-client-secret {{ CLIENT_SECRET }} - --oidc-refresh-token {{ REFRESH_TOKEN }} - --auth-file /usr/local/ec3/auth.dat &> /var/log/refresh.log" - user: root - cron_file: refresh_token - state: present - when: docker_installed.rc not in [ 1 ] -@end -) -""" - - -@click.group() -def ec3(): - """ - EC3 cluster provisioning - """ - - -@ec3.command() -@oidc_params -@auth_file_params -def refresh( - access_token, - auth_file, -): - """ - Refresh token in EC3 authorization file - """ - # Get the right endpoint from GOCDB - auth_file_contents = [] - with open(auth_file, "r") as file: - for raw_line in file.readlines(): - line = raw_line.strip() - if "OpenStack" in line: - auth_tokens = [] - for token in line.split(";"): - if token.strip().startswith("password"): - current_access_token = token.split("=")[1].strip() - if current_access_token[0] in ["'", '"']: - current_access_token = current_access_token[1:-1] - # FIXME(enolfc): add verification - payload = jwt.decode( - current_access_token, options={"verify_signature": False} - ) - now = int(time.time()) - expires = int(payload["exp"]) - if expires - now < __MIN_EXPIRATION_TIME: - current_access_token = access_token - auth_tokens.append(f"password = {current_access_token}") - else: - auth_tokens.append(token.strip()) - auth_file_contents.append("; ".join(auth_tokens)) - elif line: - auth_file_contents.append(line) - - with open(auth_file, "w+") as file: - file.write("\n".join(auth_file_contents)) - - -@ec3.command() -@site_vo_params -@oidc_params -@auth_file_params -@click.option( - "--template-dir", - help="EC3 templates dir", - default="./templates", - show_default=True, -) -@click.option("--force", is_flag=True, help="Force rewrite of files") -def init( - access_token, - site, - vo, - auth_file, - template_dir, - force, -): - """ - Create EC3 authorization file and template - """ - if os.path.exists(auth_file) and not force: - print( - "Auth file already exists, not replacing unless --force option is included" - ) - raise click.Abort() - - if site in ALL_SITES_KEYWORDS: - print("EC3 commands cannot be used with ALL_SITES") - raise click.Abort() - - endpoint, project_id, protocol = find_endpoint_and_project_id(site, vo) - site_auth = [ - f"id = {site}", - "type = OpenStack", - "username = egi.eu", - f"tenant = {protocol}", - "auth_version = 3.x_oidc_access_token", - f"host = {endpoint}", - f"domain = {project_id}", - f"password = {access_token}", - ] - auth_file_contents = [";".join(site_auth)] - - if os.path.exists(auth_file): - with open(auth_file, "r") as file: - for line in file.readlines(): - if "OpenStack" in line: - continue - auth_file_contents.append(line) - - with open(auth_file, "w+") as file: - file.write("\n".join(auth_file_contents)) - - if not os.path.exists(template_dir): - os.mkdir(template_dir) - - # FIXME: this should not be used at all! - with open(os.path.join(template_dir, "refresh.radl"), "w+") as file: - token = dict( # nosec - client_id="ADD_CLIENT_ID_HERE", - client_secret="ADD_CLIENT_SECRET_HERE", - refresh_token="ADD_REFRESH_TOKEN_HERE", - ) - file.write(EC3_REFRESHTOKEN_TEMPLATE % token) diff --git a/fedcloudclient/endpoint.py b/fedcloudclient/endpoint.py index 7160ed9..53fe6a2 100644 --- a/fedcloudclient/endpoint.py +++ b/fedcloudclient/endpoint.py @@ -283,9 +283,9 @@ def vos( project_list, project_error_list = get_projects_from_sites(access_token, site) if len(project_list) > 0: - for p in project_list: - vo = find_vo_from_project_id(p[3], p[0]) - p.insert(0, vo) + for project in project_list: + vo = find_vo_from_project_id(project[3], project[0]) + project.insert(0, vo) print( tabulate( project_list, headers=["VO", "id", "Project name", "enabled", "site"] diff --git a/fedcloudclient/exception.py b/fedcloudclient/exception.py new file mode 100644 index 0000000..1bd3223 --- /dev/null +++ b/fedcloudclient/exception.py @@ -0,0 +1,27 @@ +""" +Define custom exceptions for fedcloudclient +""" + + +class FedcloudError(Exception): + """Master class for all custom exceptions in fedcloudclient.""" + def __init__(self, message="An unspecified Fedcloud error occurred"): + super().__init__(message) + + +class TokenError(FedcloudError): + """Authentication error, token not initialized or recognized""" + def __init__(self, message="Authentication token error"): + super().__init__(message) + + +class ServiceError(FedcloudError): + """Connection timeout, service not available and so on.""" + def __init__(self, message="Service communication error"): + super().__init__(message) + + +class ConfigError(FedcloudError): + """Configuration error, file does not exist and so on.""" + def __init__(self, message="Configuration error"): + super().__init__(message) diff --git a/fedcloudclient/locker_auth.py b/fedcloudclient/locker_auth.py new file mode 100644 index 0000000..d973682 --- /dev/null +++ b/fedcloudclient/locker_auth.py @@ -0,0 +1,87 @@ +""" +Class for managing Vault locker tokens +""" +import requests + +from fedcloudclient.conf import CONF +from fedcloudclient.exception import ConfigError, ServiceError, TokenError +from fedcloudclient.logger import log_and_raise +from fedcloudclient.vault_auth import VaultToken + + +class LockerToken(VaultToken): + """ + Managing Vault locker token + """ + + def __init__(self, locker_token: str): + """ + Init Locker token + :param locker_token: + """ + if locker_token: + super().__init__(vault_token=locker_token) + else: + log_and_raise("Locker token cannot be empty", TokenError) + + def get_user_id(self): + """ + User ID is not available for locker token + :return: + """ + log_and_raise("User ID is not available for locker token", TokenError) + + def get_vault_user_id(self): + """ + Get Vault user ID from Vault token + :return: + """ + log_and_raise("User ID is not available for locker token", TokenError) + + def get_vault_auth_method(self): + """ + Get Vault user ID from Vault token + :return: + """ + log_and_raise("Auth method is not available for locker token", TokenError) + + def vault_command(self, command: str, path: str, data: dict, vo: str = None): + """ + Perform Vault command + :param command: + :param path: + :param data: + :param vo: + :return: + """ + if vo: + log_and_raise("VO-shared is not supported by locker token", TokenError) + + try: + headers = {"X-Vault-Token": self.vault_token} + url = CONF.get("vault_endpoint") + CONF.get("vault_locker_mount_point") + path + if command == "list": + response = requests.get(url, headers=headers, params={"list": "true"}) + elif command == "get": + response = requests.get(url, headers=headers) + elif command == "delete": + response = requests.delete(url, headers=headers) + elif command == "put": + response = requests.post(url, headers=headers, data=data) + else: + msg_err=f"Invalid command {command}" + log_and_raise(msg_err, ConfigError) + response=None + + if response is not None: + response.raise_for_status() + + if command in ["list", "get"]: + response_json = response.json() + return dict(response_json) + + return None + except requests.exceptions.HTTPError as exception: + error_msg = f"Error: Error when accessing secrets on server. Server response: {type(exception).__name__}: {exception}" + log_and_raise(error_msg, ServiceError) + return None diff --git a/fedcloudclient/locker_auth_test.py b/fedcloudclient/locker_auth_test.py new file mode 100644 index 0000000..b4399e4 --- /dev/null +++ b/fedcloudclient/locker_auth_test.py @@ -0,0 +1,20 @@ +""" +Testing vault_auth.py +""" +import os + +import fedcloudclient.locker_auth as locker + + +def test_get_locker_secret(locker_token: str): + """ + Test getting VO-shared secrets + """ + token = locker.LockerToken(locker_token=locker_token) + response = token.vault_command(command="read_secret", path="test", data={}, vo=None) + assert response["data"]["test"] == "test" + + +if __name__ == "__main__": + locker_token_main = os.environ["FEDCLOUD_LOCKER_TOKEN"] + test_get_locker_secret(locker_token_main) diff --git a/fedcloudclient/logger.py b/fedcloudclient/logger.py new file mode 100644 index 0000000..b44067f --- /dev/null +++ b/fedcloudclient/logger.py @@ -0,0 +1,40 @@ +""" +Logger configuration +""" + +import logging +import logging.config +from pathlib import Path + +from fedcloudclient.conf import CONF + + +def init_logger(): + """ + Init logger + :return: + """ + log_file = CONF.get("log_file") + log_level = CONF.get("log_level") + Path(log_file).parent.mkdir(parents=True, exist_ok=True) + try: + logging.config.fileConfig( + fname=CONF.get("log_config_file"), + disable_existing_loggers=False, + defaults={"log_file": log_file, "log_level": log_level}, + ) + except (ValueError, TypeError, AttributeError, ImportError, KeyError, FileNotFoundError): + logging.basicConfig(filename=log_file, level=logging.getLevelName(log_level)) + + +def log_and_raise(error_msg: str, exception): + """ + Log error and raise exception + """ + LOG.error(error_msg) + raise exception(error_msg) + + +init_logger() + +LOG = logging.getLogger("fedcloudclient") diff --git a/fedcloudclient/openstack.py b/fedcloudclient/openstack.py index f76b3c1..e4462e0 100644 --- a/fedcloudclient/openstack.py +++ b/fedcloudclient/openstack.py @@ -9,14 +9,12 @@ import subprocess # nosec Subprocess is required for invoking openstack client import sys from distutils.spawn import find_executable - import click +from fedcloudclient.logger import log_and_raise +from fedcloudclient.conf import CONF from fedcloudclient.decorators import ( ALL_SITES_KEYWORDS, - DEFAULT_AUTH_TYPE, - DEFAULT_IDENTITY_PROVIDER, - DEFAULT_PROTOCOL, all_site_params, oidc_params, openstack_output_format_params, @@ -29,6 +27,10 @@ list_sites, ) +DEFAULT_AUTH_TYPE = CONF.get("os_auth_type") +DEFAULT_IDENTITY_PROVIDER = CONF.get("os_identity_provider") +DEFAULT_PROTOCOL = CONF.get("os_protocol") + __OPENSTACK_CLIENT = "openstack" __MAX_WORKER_THREAD = 30 __MISSING_VO_ERROR_CODE = 11 @@ -38,9 +40,9 @@ def fedcloud_openstack_full( oidc_access_token, - openstack_auth_protocol, - openstack_auth_type, - checkin_identity_provider, + os_protocol, + os_auth_type, + os_identity_provider, site, vo, openstack_command, @@ -52,11 +54,12 @@ def fedcloud_openstack_full( :param oidc_access_token: Checkin access token. Passed to openstack client as --os-access-token - :param openstack_auth_protocol: Checkin protocol (openid, oidc). Passed to + :param os_protocol: Checkin protocol (openi + , oidc). Passed to openstack client as --os-protocol - :param openstack_auth_type: Checkin authentication type (v3oidcaccesstoken). + :param os_auth_type: Checkin authentication type (v3oidcaccesstoken). Passed to openstack client as --os-auth-type - :param checkin_identity_provider: Checkin identity provider in mapping (egi.eu). + :param os_identity_provider: Checkin identity provider in mapping (egi.eu). Passed to openstack client as --os-identity-provider :param site: site ID in GOCDB :param vo: VO name @@ -65,6 +68,7 @@ def fedcloud_openstack_full( :param json_output: if result is JSON object or string. Default:True :return: error code, result or error message + """ endpoint, project_id, protocol = find_endpoint_and_project_id(site, vo) @@ -72,17 +76,17 @@ def fedcloud_openstack_full( return __MISSING_VO_ERROR_CODE, f"VO {vo} not found on site {site}\n" if protocol is None: - protocol = openstack_auth_protocol + protocol = os_protocol options = ( "--os-auth-url", endpoint, "--os-auth-type", - openstack_auth_type, + os_auth_type, "--os-protocol", protocol, "--os-identity-provider", - checkin_identity_provider, + os_identity_provider, "--os-access-token", oidc_access_token, ) @@ -105,7 +109,7 @@ def fedcloud_openstack_full( (__OPENSTACK_CLIENT,) + openstack_command + options, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=my_env, + env=my_env, check=True ) error_code = completed.returncode @@ -146,6 +150,7 @@ def fedcloud_openstack( :param json_output: if result is JSON object or string. Default:True :return: error code, result or error message + """ return fedcloud_openstack_full( @@ -194,6 +199,7 @@ def print_result( :param ignore_missing_vo: :param first: :return: + """ command = " ".join(command) @@ -238,14 +244,15 @@ def openstack( all_sites, vo, openstack_command, - openstack_auth_protocol, - openstack_auth_type, - openstack_auth_provider, + os_protocol, + os_auth_type, + os_identity_provider, ignore_missing_vo, json_output, ): """ Execute OpenStack commands on site and VO + """ if not check_openstack_client_installation(): @@ -265,9 +272,9 @@ def openstack( executor.submit( fedcloud_openstack_full, access_token, - openstack_auth_protocol, - openstack_auth_type, - openstack_auth_provider, + os_protocol, + os_auth_type, + os_identity_provider, site, vo, openstack_command, @@ -279,14 +286,16 @@ def openstack( # Get results and print them first = True - # Get the result, first come first serve + # Get the result, first come, first served for future in concurrent.futures.as_completed(results): site = results[future] exc_msg = None try: error_code, result = future.result() - except Exception as exc: - exc_msg = exc + except Exception as exception: + msg_err=f"Can not get result in OpenStack: {exception}" + log_and_raise(msg_err, exception) + raise Exception(msg_err) from exception # Print result print_result( @@ -316,12 +325,15 @@ def openstack_int( access_token, site, vo, - openstack_auth_protocol, - openstack_auth_type, - openstack_auth_provider, -): + os_protocol, + os_auth_type, + os_identity_provider, + ): """ Interactive OpenStack client on site and VO + + :return: None + """ if not check_openstack_client_installation(): @@ -332,15 +344,16 @@ def openstack_int( raise SystemExit(f"Error: VO {vo} not found on site {site}") if protocol is None: - protocol = openstack_auth_protocol + protocol = os_protocol my_env = os.environ.copy() my_env["OS_AUTH_URL"] = endpoint - my_env["OS_AUTH_TYPE"] = openstack_auth_type + my_env["OS_AUTH_TYPE"] = os_auth_type my_env["OS_PROTOCOL"] = protocol - my_env["OS_IDENTITY_PROVIDER"] = openstack_auth_provider + my_env["OS_IDENTITY_PROVIDER"] = os_identity_provider my_env["OS_ACCESS_TOKEN"] = access_token my_env["OS_PROJECT_ID"] = project_id # Calling OpenStack client as subprocess # Ignore bandit warning - subprocess.run(__OPENSTACK_CLIENT, env=my_env) # nosec + subprocess.run(__OPENSTACK_CLIENT, env=my_env, check=True) # nosec + return None diff --git a/fedcloudclient/secret.py b/fedcloudclient/secret.py index 113b501..5d35f80 100644 --- a/fedcloudclient/secret.py +++ b/fedcloudclient/secret.py @@ -1,34 +1,26 @@ """ Implementation of "fedcloud secret" commands for accessing secret management service """ -import base64 -import json -import os import sys import click import hvac import requests -import yaml -from cryptography.fernet import Fernet, InvalidToken -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC from hvac.exceptions import VaultError -from tabulate import tabulate -from yaml import YAMLError -from fedcloudclient.checkin import get_checkin_id -from fedcloudclient.decorators import ( - oidc_params, - secret_output_params, - secret_token_params, -) +from fedcloudclient.auth import OIDCToken +from fedcloudclient.conf import CONF +from fedcloudclient.decorators import oidc_params, secret_output_params, secret_token_params +from fedcloudclient.logger import LOG, log_and_raise +from fedcloudclient.secret_helper import decrypt_data, encrypt_data, print_secrets, print_value, secret_params_to_dict +from fedcloudclient.vault_auth import VaultToken +from fedcloudclient.exception import ServiceError -VAULT_ADDR = "https://vault.services.fedcloud.eu:8200" -VAULT_ROLE = "" -VAULT_MOUNT_POINT = "/secrets/" -VAULT_SALT = "fedcloud_salt" -VAULT_LOCKER_MOUNT_POINT = "/v1/cubbyhole/" +VAULT_ADDR = CONF.get("vault_endpoint") +VAULT_ROLE = CONF.get("vault_role") +VAULT_MOUNT_POINT = CONF.get("vault_mount_point") +VAULT_SALT = CONF.get("vault_salt") +VAULT_LOCKER_MOUNT_POINT = CONF.get("vault_locker_mount_point") def secret_client(access_token, command, path, data): @@ -44,7 +36,8 @@ def secret_client(access_token, command, path, data): try: client = hvac.Client(url=VAULT_ADDR) client.auth.jwt.jwt_login(role=VAULT_ROLE, jwt=access_token) - checkin_id = get_checkin_id(access_token) + token=OIDCToken() + checkin_id = token.get_checkin_id(access_token) full_path = "users/" + checkin_id + "/" + path function_list = { "list_secrets": client.secrets.kv.v1.list_secrets, @@ -63,10 +56,10 @@ def secret_client(access_token, command, path, data): mount_point=VAULT_MOUNT_POINT, ) return response - except VaultError as e: - raise SystemExit( - f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" - ) + except VaultError as err: + err_msg=f"Error: Error when accessing secrets on server. Server response: {type(err).__name__}: {err}" + log_and_raise(err_msg, err) + raise SystemExit(err_msg) from err def locker_client(locker_token, command, path, data): @@ -96,183 +89,11 @@ def locker_client(locker_token, command, path, data): if command in ["list_secrets", "read_secret"]: response_json = response.json() return dict(response_json) - else: - return None + return None except requests.exceptions.HTTPError as exception: - raise SystemExit( - f"Error: Error when accessing secrets on server. Server response: {type(exception).__name__}: {exception}" - ) - - -def read_data_from_file(input_format, input_file): - """ - Read data from file. Format may be text, yaml, json or auto-detect according to file extension - :param input_format: - :param input_file: - :return: - """ - - if input_format is None or input_format == "auto-detect": - if input_file.endswith(".json"): - input_format = "json" - else: - # default format - input_format = "yaml" - - try: - - # read text/binary files to strings - if input_format == "binary": - with open(input_file, "rb") if input_file else sys.stdin.buffer as f: - return base64.b64encode(f.read()).decode() - if input_format == "text": - with open(input_file, "r") if input_file else sys.stdin as f: - return f.read() - - # reading YAML or JSON to dict - with open(input_file) if input_file else sys.stdin as f: - if input_format == "yaml": - data = yaml.safe_load(f) - elif input_format == "json": - data = json.load(f) - return dict(data) - - except (ValueError, FileNotFoundError, YAMLError) as e: - raise SystemExit( - f"Error: Error when reading file {input_file}. Error message: {type(e).__name__}: {e}" - ) - - -def secret_params_to_dict(params, binary_file=False): - """ - Convert secret params "key=value" to dict {"key":"value"} - :param binary_file: if reading files as binary - :param params: input string in format "key=value" - :return: dict {"key":"value"} - """ - - result = {} - - if len(params) == 0: - raise SystemExit( - "Error: Expecting 'key=value' arguments for secrets, None provided." - ) - - for param in params: - if param.startswith("@") or param == "-": - data = read_data_from_file(None, param[1:]) - result.update(data) - else: - try: - key, value = param.split("=", 1) - except ValueError: - raise SystemExit( - f"Error: Expecting 'key=value' arguments for secrets. '{param}' provided." - ) - if value.startswith("@") or value == "-": - if binary_file: - value = read_data_from_file("binary", value[1:]) - else: - value = read_data_from_file("text", value[1:]) - result[key] = value - - return result - - -def generate_derived_key(salt, passphrase): - """ - Generate derived encryption/decryption key from salted passphrase - :param salt: - :param passphrase: - :return: derived key - """ - - kdf = PBKDF2HMAC( - algorithm=hashes.SHA256(), - length=32, - salt=salt, - iterations=390000, - ) - return base64.b64encode(kdf.derive(passphrase.encode())) - - -def encrypt_data(encrypt_key, secrets): - """ - Encrypt values in secrets using key - :param encrypt_key: encryption key - :param secrets: dict containing secrets - :return: dict with encrypted values - """ - salt = os.urandom(16) - derived_key = generate_derived_key(salt, encrypt_key) - fernet = Fernet(derived_key) - for key in secrets: - secrets[key] = fernet.encrypt(secrets[key].encode()) - secrets[VAULT_SALT] = base64.b64encode(salt) - - -def decrypt_data(decrypt_key, secrets): - """ - Decrypt values in secrets using key - :param decrypt_key: decryption key - :param secrets: dict containing encrypted secrets - :return: dict with decrypted values - """ - try: - salt = base64.b64decode(secrets.pop(VAULT_SALT)) - derived_key = generate_derived_key(salt, decrypt_key) - fernet = Fernet(derived_key) - for key in secrets: - secrets[key] = fernet.decrypt(secrets[key].encode()).decode() - except InvalidToken as e: - raise SystemExit(f"Error: Error during decryption. {e}") - - -def print_secrets(output_file, output_format, secrets): - """ - Print secrets in different formats - :param output_file: - :param output_format: - :param secrets: - :return: - """ - - try: - with open(output_file, "wt") if output_file else sys.stdout as f: - if output_format == "JSON": - json.dump(secrets, f, indent=4) - elif output_format == "YAML": - yaml.dump(secrets, f, sort_keys=False) - else: - print(tabulate(secrets.items(), headers=["key", "value"]), file=f) - - except (ValueError, FileNotFoundError, YAMLError) as e: - raise SystemExit( - f"Error: Error when writing file {output_file}. Error message: {type(e).__name__}: {e}" - ) - - -def print_value(output_file, binary_file, value): - """ - Print secrets in different formats - :param output_file: - :param binary_file: - :param value: - :return: - """ - - try: - if binary_file: - with open(output_file, "wb") if output_file else sys.stdout.buffer as f: - f.write(base64.b64decode(value.encode())) - else: - with open(output_file, "wt") if output_file else sys.stdout as f: - f.write(value) - - except (ValueError, FileNotFoundError, TypeError) as e: - raise SystemExit( - f"Error: Error when writing file {output_file}. Error message: {type(e).__name__}: {e}" - ) + err_msg=f"Error: Error when accessing secrets on server. Server response: {type(exception).__name__}: {exception}" + log_and_raise(err_msg, exception) + raise SystemExit(err_msg) from exception @click.group() @@ -285,12 +106,12 @@ def secret(): @secret.command() @secret_token_params @secret_output_params -@click.argument("short_path", metavar="[secret path]") +@click.argument("short_path", metavar="[secret_path]") @click.argument("key", metavar="[key]", required=False) @click.option( "--decrypt-key", "-d", - metavar="[key]", + metavar="passphrase", required=False, help="Decryption key or passphrase", ) @@ -304,54 +125,63 @@ def secret(): @click.option( "--output-file", "-o", - metavar="[filename]", + metavar="filename", required=False, help="Name of output file", ) def get( - access_token, - locker_token, - short_path, - key, - output_format, - decrypt_key, - binary_file, - output_file, + token: VaultToken, + short_path: str, + key: str, + output_format: str, + decrypt_key: str, + binary_file: bool, + output_file: str, ): """ Get the secret object in the path. If a key is given, print only the value of the key """ - if locker_token: - response = locker_client(locker_token, "read_secret", short_path, None) - else: - response = secret_client(access_token, "read_secret", short_path, None) - if decrypt_key: - decrypt_data(decrypt_key, response["data"]) - if not key: - print_secrets(output_file, output_format, response["data"]) - else: - if key in response["data"]: - print_value(output_file, binary_file, response["data"][key]) - else: - raise SystemExit(f"Error: {key} not found in {short_path}") + try: + response = token.vault_command(command="get", path=short_path, data={}) + if decrypt_key: + decrypt_data(decrypt_key, response["data"]) + if not key: + print_secrets(output_file, output_format, response["data"]) + else: + if key in response["data"]: + print_value(output_file, binary_file, response["data"][key]) + else: + raise SystemExit(f"Error: {key} not found in {short_path}") + except Exception as exception: + msg_err=f"An unexpected error occurred: {str(exception)}" #, file=sys.stderr + log_and_raise(msg_err, ServiceError(msg_err)) + raise ServiceError(msg_err) from exception @secret.command("list") @secret_token_params @click.argument("short_path", metavar="[secret path]", required=False, default="") def list_( - access_token, - locker_token, - short_path, + token: VaultToken, + short_path: str, ): """ List secret objects in the path """ - if locker_token: - response = locker_client(locker_token, "list_secrets", short_path, None) - else: - response = secret_client(access_token, "list_secrets", short_path, None) - print("\n".join(map(str, response["data"]["keys"]))) + try: + response = token.vault_command(command="list", path=short_path, data={}) + print("\n".join(map(str, response["data"]["keys"]))) + except Exception as exception: + message = str(exception) + if "HTTPError: 404" in message: + file=sys.stderr + msg_err=f"No secrets found: {file}" + log_and_raise(msg_err, ServiceError(msg_err)) + raise ServiceError(msg_err) from exception + + msg_err=f"An unexpected error occurred: {str(exception)}"#, file=sys.stderr + log_and_raise(msg_err, ServiceError(msg_err)) + raise ServiceError(msg_err) from exception @secret.command() @@ -361,7 +191,7 @@ def list_( @click.option( "--encrypt-key", "-e", - metavar="[key]", + metavar="passphrase", help="Encryption key or passphrase", ) @click.option( @@ -371,12 +201,11 @@ def list_( help="True for reading secrets from binary files", ) def put( - access_token, - locker_token, - short_path, - secrets, - encrypt_key, - binary_file, + token: VaultToken, + short_path: str, + secrets: list, + encrypt_key: str, + binary_file: bool, ): """ Put a secret object to the path. Secrets are provided in form key=value @@ -385,27 +214,20 @@ def put( secret_dict = secret_params_to_dict(secrets, binary_file) if encrypt_key: encrypt_data(encrypt_key, secret_dict) - if locker_token: - locker_client(locker_token, "put", short_path, secret_dict) - else: - secret_client(access_token, "put", short_path, secret_dict) + token.vault_command(command="put", path=short_path, data=secret_dict) @secret.command() @secret_token_params @click.argument("short_path", metavar="[secret path]") def delete( - access_token, - locker_token, - short_path, + token: VaultToken, + short_path, ): """ Delete the secret object in the path """ - if locker_token: - locker_client(locker_token, "delete_secret", short_path, None) - else: - secret_client(access_token, "delete_secret", short_path, None) + token.vault_command(command="delete", path=short_path, data={}) @secret.group() @@ -418,13 +240,14 @@ def locker(): @locker.command() @oidc_params @secret_output_params -@click.option("--ttl", default="24h", help="Time-to-live for the new locker") -@click.option("--num-uses", default=10, help="Max number of uses") +@click.option("--ttl", default="24h", help="Locker's Time-to-live", show_default=True) +@click.option("--num-uses", default=10, help="Max number of uses", show_default=True) @click.option("--verbose", is_flag=True, help="Print token details") def create(access_token, ttl, num_uses, output_format, verbose): """ Create a locker and return the locker token """ + LOG.debug("Creating a new locker") try: client = hvac.Client(url=VAULT_ADDR) client.auth.jwt.jwt_login(role=VAULT_ROLE, jwt=access_token) @@ -435,11 +258,11 @@ def create(access_token, ttl, num_uses, output_format, verbose): if not verbose: print(locker_token["auth"]["client_token"]) else: - print_secrets(None, output_format, locker_token["auth"]) - except VaultError as e: - raise SystemExit( - f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" - ) + print_secrets("", output_format, locker_token["auth"]) + except VaultError as exception: + msg_err=f"Error: Error when accessing secrets on server. Server response: {type(exception).__name__}: {exception}" + log_and_raise(msg_err, ServiceError(msg_err)) + raise SystemExit(msg_err) from exception @locker.command() @@ -456,11 +279,11 @@ def check(locker_token, output_format): client = hvac.Client(url=VAULT_ADDR) client.token = locker_token locker_info = client.auth.token.lookup_self() - print_secrets(None, output_format, locker_info["data"]) - except VaultError as e: - raise SystemExit( - f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" - ) + print_secrets("", output_format, locker_info["data"]) + except VaultError as exception: + msg_err=f"Error: Error when accessing secrets on server. Server response: {type(exception).__name__}: {exception}" + log_and_raise(msg_err, ServiceError(msg_err)) + raise SystemExit(msg_err) from exception @locker.command() @@ -475,7 +298,7 @@ def revoke(locker_token): client = hvac.Client(url=VAULT_ADDR) client.token = locker_token client.auth.token.revoke_self() - except VaultError as e: - raise SystemExit( - f"Error: Error when accessing secrets on server. Server response: {type(e).__name__}: {e}" - ) + except VaultError as exception: + msg_err=f"Error: Error when accessing secrets on server. Server response: {type(exception).__name__}: {exception}" + log_and_raise(msg_err, ServiceError(msg_err)) + raise SystemExit(msg_err) from exception diff --git a/fedcloudclient/secret_helper.py b/fedcloudclient/secret_helper.py new file mode 100644 index 0000000..3d00de4 --- /dev/null +++ b/fedcloudclient/secret_helper.py @@ -0,0 +1,196 @@ +""" +Secret helper functions +""" +import base64 +import json +import os +import sys + +import yaml +from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC +from tabulate import tabulate + +from fedcloudclient.conf import CONF +from fedcloudclient.exception import ServiceError +from fedcloudclient.logger import log_and_raise + +VAULT_SALT = CONF.get("vault_salt") + + +def read_data_from_file(input_format: str, input_file: str): + """ + Read data from file. Format may be text, yaml, json or auto-detect according to file extension + :param input_format: + :param input_file: + :return: + """ + + if input_format is None or input_format == "auto-detect": + if input_file.endswith(".json"): + input_format = "JSON" + else: + # default format + input_format = "YAML" + + try: + + # read text/binary files to strings + if input_format == "binary": + with open(input_file, "rb") if input_file else sys.stdin.buffer as file: + return base64.b64encode(file.read()).decode() + if input_format == "text": + with open(input_file, "r", encoding="utf-8") if input_file else sys.stdin as file: + return file.read() + + # reading YAML or JSON to dict + with open(input_file, encoding="utf-8") if input_file else sys.stdin as file: + data = {} + if input_format == "YAML": + data = yaml.safe_load(file) + elif input_format == "JSON": + data = json.load(file) + return dict(data) + + except (ValueError, FileNotFoundError, yaml.YAMLError) as exception: + msg_err= f"Error: Error when reading file {input_file}. Error message: {type(exception).__name__}: {exception}" + log_and_raise(msg_err, ServiceError(msg_err)) + raise SystemExit(msg_err) from exception + + + +def secret_params_to_dict(params: list, binary_file: bool = False): + """ + Convert secret params "key=value" to dict {"key":"value"} + :param binary_file: if reading files as binary + :param params: input string in format "key=value" + :return: dict {"key":"value"} + """ + + result = {} + + if len(params) == 0: + raise SystemExit( + "Error: Expecting 'key=value' arguments for secrets, None provided." + ) + + for param in params: + if param.startswith("@") or param == "-": + data = read_data_from_file("auto-detect", param[1:]) + result.update(data) + else: + try: + key, value = param.split("=", 1) + except ValueError as exception: + msg_err=f"Error: Expecting 'key=value' arguments for secrets. '{param}' provided." + log_and_raise(msg_err, ServiceError(msg_err)) + raise SystemExit(msg_err) from exception + + if value.startswith("@") or value == "-": + if binary_file: + value = read_data_from_file("binary", value[1:]) + else: + value = read_data_from_file("text", value[1:]) + result[key] = value + + return result + + +def generate_derived_key(salt: bytes, passphrase: str): + """ + Generate derived encryption/decryption key from salted passphrase + :param salt: + :param passphrase: + :return: derived key + """ + + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + iterations=390000, + ) + return base64.b64encode(kdf.derive(passphrase.encode())) + + +def encrypt_data(encrypt_key: str, secrets: dict): + """ + Encrypt values in secrets using key + :param encrypt_key: encryption key + :param secrets: dict containing secrets + :return: dict with encrypted values + """ + salt = os.urandom(16) + derived_key = generate_derived_key(salt, encrypt_key) + fernet = Fernet(derived_key) + for key in secrets: + secrets[key] = fernet.encrypt(secrets[key].encode()).decode() + secrets[VAULT_SALT] = base64.b64encode(salt) + + +def decrypt_data(decrypt_key: str, secrets: dict): + """ + Decrypt values in secrets using key + :param decrypt_key: decryption key + :param secrets: dict containing encrypted secrets + :return: dict with decrypted values + """ + try: + salt = base64.b64decode(secrets.pop(VAULT_SALT)) + derived_key = generate_derived_key(salt, decrypt_key) + fernet = Fernet(derived_key) + for key in secrets: + secrets[key] = fernet.decrypt(secrets[key].encode()).decode() + except InvalidToken as exception: + msg_err=f"Error: Error during decryption. {exception}" + log_and_raise(msg_err, ServiceError(msg_err)) + raise SystemExit(msg_err) from exception + + +def print_secrets(output_file: str, output_format: str, secrets: dict): + """ + Print secrets in different formats + :param output_file: + :param output_format: + :param secrets: + :return: + """ + + try: + with open(output_file, "wt", encoding="utf-8") if output_file else sys.stdout as file: + if output_format == "JSON": + json.dump(secrets, file, indent=4) + elif output_format == "YAML": + yaml.dump(secrets, file, sort_keys=False) + else: + print(tabulate(secrets.items(), headers=["key", "value"]), file=file) + + except (ValueError, FileNotFoundError, yaml.YAMLError) as exception: + msg_err= f"Error: Error when writing file {output_file}. Error message: {type(exception).__name__}: {exception}" + log_and_raise(msg_err, ServiceError(msg_err)) + raise SystemExit(msg_err) from exception + + + +def print_value(output_file: str, binary_file: bool, value: str): + """ + Print secrets in different formats + :param output_file: + :param binary_file: + :param value: + :return: + """ + + try: + if binary_file: + with open(output_file, "wb", encoding="utf-8") if output_file else sys.stdout.buffer as file: + file.write(base64.b64decode(value.encode())) + else: + with open(output_file, "wt", encoding="utf-8") if output_file else sys.stdout as file: + file.write(value) + + except (ValueError, FileNotFoundError, TypeError) as exception: + msg_err=f"Error: Error when writing file {output_file}. Error message: {type(exception).__name__}: {exception}" + log_and_raise(msg_err, ServiceError(msg_err)) + raise SystemExit(msg_err) from exception diff --git a/fedcloudclient/secret_helper_test.py b/fedcloudclient/secret_helper_test.py new file mode 100644 index 0000000..36937aa --- /dev/null +++ b/fedcloudclient/secret_helper_test.py @@ -0,0 +1,67 @@ +""" +Testing secret helper functions +""" +import base64 +import json +import os + +from fedcloudclient.secret_helper import decrypt_data, encrypt_data, print_secrets, print_value, read_data_from_file + +def save_read_binary_files(): + """ + Test save random binary data, read it again and compare + """ + print("Testing read/write data from binary files") + data = base64.b64encode(os.urandom(20)).decode() + temp_file = "/tmp/test" + print_value(output_file=temp_file, binary_file=True, value=data) + read_data = read_data_from_file(input_format="binary", input_file=temp_file) + + print(f" Original : {data} \n from file: {read_data}") + assert read_data == data + + +def save_read_dict(): + """ + Test saving dict in temp file and read it again + """ + print("Testing read/write data from YAML/JSON files") + + data = {"key1": "value1", "key2": base64.b64encode(os.urandom(8)).decode()} + temp_file = "/tmp/test" + print_secrets(output_file=temp_file, output_format="JSON", secrets=data) + save_data = read_data_from_file(input_format="JSON", input_file=temp_file) + + print(f" Original : {json.dumps(data)} \n from file: {json.dumps(save_data)}") + + assert data == save_data + print_secrets(output_file=temp_file, output_format="YAML", secrets=data) + save_data = read_data_from_file(input_format="YAML", input_file=temp_file) + + print(f" Original : {json.dumps(data)} \n from file: {json.dumps(save_data)}") + + assert data == save_data + + +def encrypt_decrypt(): + """ + Testing encrypt/decrypt + """ + print("Testing encrypt/decrypt") + secret = base64.b64encode(os.urandom(20)).decode() + passphrase = base64.b64encode(os.urandom(8)).decode() + data = {"key": secret} + encrypt_data(encrypt_key=passphrase, secrets=data) + print(f" Original : {secret} \n encrypted: {data['key']}") + decrypt_data(decrypt_key=passphrase, secrets=data) + print(f" Original : {secret} \n decrypted: {data['key']}") + assert secret == data["key"] + + +if __name__ == "__main__": + + print("Test help function") + + save_read_binary_files() + save_read_dict() + encrypt_decrypt() diff --git a/fedcloudclient/select.py b/fedcloudclient/select.py index 8fa8d5b..f057c8b 100644 --- a/fedcloudclient/select.py +++ b/fedcloudclient/select.py @@ -20,6 +20,7 @@ ) from fedcloudclient.openstack import fedcloud_openstack from fedcloudclient.sites import find_endpoint_and_project_id +from fedcloudclient.logger import log_and_raise FILTER_TEMPLATE = "$[?( {specs} )]" GET_FLAVOR_COMMAND = ("flavor", "list", "--long") @@ -109,11 +110,12 @@ def get_parser(filter_string): try: parser = parse(filter_string) except JSONPathError as exception: - raise SystemExit( - "Error during constructing filter\n" + msg_err="""Error during constructing filter\n" f"Filter string: {filter_string}\n" - f"{exception}" - ) + f"{exception}""" + log_and_raise(msg_err, JSONPathError) + raise SystemExit(msg_err) from exception + return parser @@ -129,11 +131,11 @@ def do_filter(parser, input_list): try: matched = [match.value for match in parser.find(input_list)] except TypeError as exception: - raise SystemExit( - "TypeError during filtering result\n" + msg_err="""TypeError during filtering result\n" "Probably string value for numeric property in filter\n" - f"{exception}" - ) + f"{exception}""" + log_and_raise(msg_err, TypeError) + raise SystemExit(msg_err) from exception return matched diff --git a/fedcloudclient/shell.py b/fedcloudclient/shell.py index 88b2413..e93bdc7 100644 --- a/fedcloudclient/shell.py +++ b/fedcloudclient/shell.py @@ -35,7 +35,7 @@ def get_shell_type(): return Shell.LINUX - +#Imported to the sites def print_set_env_command(name, value): """ Print command to set environment variable, diff --git a/fedcloudclient/sites.py b/fedcloudclient/sites.py index fe491ab..8c407c4 100644 --- a/fedcloudclient/sites.py +++ b/fedcloudclient/sites.py @@ -22,9 +22,9 @@ import yaml from jsonschema import validate +from fedcloudclient.conf import CONF from fedcloudclient.decorators import ( ALL_SITES_KEYWORDS, - DEFAULT_PROTOCOL, all_site_params, oidc_params, site_vo_params, @@ -32,6 +32,8 @@ ) from fedcloudclient.shell import print_set_env_command +DEFAULT_PROTOCOL = CONF.get("os_protocol") + __REMOTE_CONFIG_FILE = ( "https://raw.githubusercontent.com/tdviet/fedcloudclient/master/config/sites.yaml" ) @@ -102,7 +104,7 @@ def safe_read_yaml_from_url(url, max_length): data = yaml.safe_load(yaml_file) except Exception as exception: print(f"Error during reading data from {url}") - raise SystemExit(f"Exception: {exception}") + raise SystemExit(f"Exception: {exception}") from exception return data @@ -130,7 +132,7 @@ def read_default_site_config(): validate(instance=site_info, schema=schema) except Exception as exception: print(f"Site config in file {filename} is in wrong format") - raise SystemExit(f"Exception: {exception}") + raise SystemExit(f"Exception: {exception}") from exception __site_config_data.append(site_info) @@ -154,7 +156,7 @@ def read_local_site_config(config_dir): __site_config_data.append(site_info) except Exception as exception: print(f"Error during reading site config from {file}") - raise SystemExit(f"Exception: {exception}") + raise SystemExit(f"Exception: {exception}") from exception def save_site_config(config_dir): @@ -274,40 +276,40 @@ def site(): @site.command() @all_site_params -def show(site, all_sites): +def show(site_local, all_sites): """ Print configuration of specified site(s) """ - if site in ALL_SITES_KEYWORDS or all_sites: + if site_local in ALL_SITES_KEYWORDS or all_sites: read_site_config() for site_info in __site_config_data: site_info_str = yaml.dump(site_info, sort_keys=True) print(site_info_str) else: - site_info = find_site_data(site) + site_info = find_site_data(site_local) if site_info: print(yaml.dump(site_info, sort_keys=True)) else: - raise SystemExit(f"Site {site} not found") + raise SystemExit(f"Site {site_local} not found") @site.command() @site_vo_params -def show_project_id(site, vo): +def show_project_id(site_local, vo): """ Print Keystone endpoint and project ID """ - if site in ALL_SITES_KEYWORDS: + if site_local in ALL_SITES_KEYWORDS: print("Cannot get project ID for ALL_SITES") raise click.Abort() - endpoint, project_id, _ = find_endpoint_and_project_id(site, vo) + endpoint, project_id, _ = find_endpoint_and_project_id(site_local, vo) if endpoint: print_set_env_command("OS_AUTH_URL", endpoint) print_set_env_command("OS_PROJECT_ID", project_id) else: - raise SystemExit(f"VO {vo} not found on site {site}") + raise SystemExit(f"VO {vo} not found on site {site_local}") @site.command() @@ -330,15 +332,15 @@ def list_(vo=None): List all sites. If "--vo " is provided, list only sites supporting a Virtual Organization. """ - for site in list_sites(vo): - print(site) + for site_local in list_sites(vo): + print(site_local) @site.command() @site_vo_params @oidc_params def env( - site, + site_local, vo, access_token, ): @@ -347,11 +349,11 @@ def env( May set also environment variable OS_ACCESS_TOKEN, if access token is provided, otherwise print notification """ - if site in ALL_SITES_KEYWORDS: + if site_local in ALL_SITES_KEYWORDS: print("Cannot generate environment variables for ALL_SITES") raise click.Abort() - endpoint, project_id, protocol = find_endpoint_and_project_id(site, vo) + endpoint, project_id, protocol = find_endpoint_and_project_id(site_local, vo) if endpoint: if protocol is None: protocol = DEFAULT_PROTOCOL @@ -362,5 +364,5 @@ def env( print_set_env_command("OS_PROJECT_ID", project_id) print_set_env_command("OS_ACCESS_TOKEN", access_token) else: - print(f"VO {vo} not found to have access to site {site}") + print(f"VO {vo} not found to have access to site {site_local}") return 1 diff --git a/fedcloudclient/vault_auth.py b/fedcloudclient/vault_auth.py new file mode 100644 index 0000000..6e5af0c --- /dev/null +++ b/fedcloudclient/vault_auth.py @@ -0,0 +1,143 @@ +""" +Class for managing Vault tokens +""" +import hvac +from hvac.exceptions import VaultError + +from fedcloudclient.auth import OIDCToken +from fedcloudclient.conf import CONF +from fedcloudclient.exception import TokenError +from fedcloudclient.logger import LOG, log_and_raise + + +class VaultToken(OIDCToken): + """ + Managing tokens for Vault + """ + + def __init__(self, access_token: str = None, vault_token: str = None): + """ + Init Vault token + """ + super().__init__(access_token) + self.vault_token = vault_token + self.auth_method = None + self.vault_client = None + + def get_vault_client(self) -> hvac.Client: + """ + Init client if needed and return the client + """ + if self.vault_client: + # client is initialized, just return it + return self.vault_client + + # Create the client + client = hvac.Client(url=CONF.get("vault_endpoint")) + if self.vault_token: + client.token = self.vault_token + self.vault_client = client + return client + if self.access_token: + try: + client.auth.jwt.jwt_login(role=CONF.get("vault_role"), jwt=self.access_token) + self.vault_token = client.token + self.vault_client = client + return client + except VaultError as exception: + error_msg = f"Cannot login to Vault via access token: {exception}" + log_and_raise(error_msg, TokenError) + + error_msg = "Token is not initialized" + log_and_raise(error_msg, TokenError) + return None + + def get_vault_token(self) -> str: + """ + Return Vault token + """ + if self.vault_token: + return self.vault_token + if self.access_token: + self.get_vault_client() + return self.vault_token + + error_msg = "Vault token is not initialized" + log_and_raise(error_msg, TokenError) + return None + def get_user_id(self) -> str: + """ + Get user ID (from access token or vault token) + """ + if self.access_token: + return super().get_user_id() + return self.get_vault_user_id() + + def get_vault_user_id(self) -> str: + """ + Get Vault user ID from Vault token + """ + response = {} + try: + client = self.get_vault_client() + response = client.auth.token.lookup_self() + except VaultError as exception: + error_msg = f"Cannot get user information via Vault token: {exception}" + log_and_raise(error_msg, TokenError) + + display_name = response["data"]["display_name"] + msg_debug=f"Vault token display_name: {display_name}" + LOG.debug(msg_debug) + self.auth_method, self.user_id = display_name.split("-") + return self.user_id + + def get_vault_auth_method(self) -> str: + """ + Get authentication method creating the token + """ + if self.auth_method: + return self.auth_method + + self.get_vault_user_id() + return self.auth_method + + def vault_command(self, command: str, path: str, data: dict, vo: str = None) -> dict: + """ + Perform Vault kv command + """ + + client = self.get_vault_client() + + full_path = "" + if vo: + if self.get_vault_auth_method() == "oidc": + full_path = "vos/" + vo + "/" + path + else: + log_and_raise("VO-shared folders are accessible only for token created by OIDC method via GUI",TokenError) + else: + full_path = "users/" + self.get_user_id() + "/" + path + + function_list = { + "list": client.secrets.kv.v1.list_secrets, + "get": client.secrets.kv.v1.read_secret, + "delete": client.secrets.kv.v1.delete_secret, + } + mount_point = CONF.get("vault_mount_point") + try: + if command == "put": + response = client.secrets.kv.v1.create_or_update_secret( + path=full_path, + mount_point=mount_point, + secret=data, + ) + else: + response = function_list[command]( + path=full_path, + mount_point=mount_point, + ) + return response + + except VaultError as exception: + error_msg = f"Error: Error when accessing secrets on server. Server response: {type(exception).__name__}: {exception}" + log_and_raise(error_msg, TokenError) + return None diff --git a/fedcloudclient/vault_auth_test.py b/fedcloudclient/vault_auth_test.py new file mode 100644 index 0000000..7f52494 --- /dev/null +++ b/fedcloudclient/vault_auth_test.py @@ -0,0 +1,65 @@ +""" +Testing vault_auth.py +""" +import os + +import fedcloudclient.vault_auth as vault +from fedcloudclient.exception import TokenError + + +def test_vault_login(mytoken: str): + """ + test vault login with mytoken + """ + + token = vault.VaultToken() + token.get_token_from_mytoken(mytoken) + vault_client = token.get_vault_client() + + assert vault_client + + +def test_user_id_from_vault_token(vault_token: str, user_id: str): + """ + Test user id from OIDC vault token + """ + token = vault.VaultToken(vault_token=vault_token) + vault_id = None + try: + vault_id = token.get_user_id() + except TokenError: + print("Please check validity of your OIDC Vault token") + assert vault_id == user_id + + +def test_get_personal_secret(vault_token: str): + """ + Test getting personal secrets + """ + token = vault.VaultToken(vault_token=vault_token) + response = token.vault_command(command="get", path="test", data={}, vo=None) + assert response["data"]["test"] == "test" + + +def test_get_vo_secret(vault_token: str, vo_secret: str): + """ + Test getting VO-shared secrets + """ + token = vault.VaultToken(vault_token=vault_token) + response = token.vault_command(command="get", path="test", data={}, vo=vo_secret) + assert response["data"]["test"] == "test" + + +if __name__ == "__main__": + #Before testing, setup testing environment with + #export FEDCLOUD_MYTOKEN= + #export FEDCLOUD_ID= + #export FEDCLOUD_VAULT_TOKEN= + + os_mytoken = os.environ["FEDCLOUD_MYTOKEN"] + os_user_id = os.environ["FEDCLOUD_ID"] + oidc_vault_token = os.environ["FEDCLOUD_VAULT_TOKEN"] + test_vault_login(os_mytoken) + test_user_id_from_vault_token(oidc_vault_token, os_user_id) + test_get_personal_secret(oidc_vault_token) + test_get_vo_secret(oidc_vault_token, "vo.access.egi.eu")