diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1acd67fd3c..64a44fb5c2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,11 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + id: extract_branch + - name: Install dependencies run: | sudo apt-get update @@ -38,9 +43,11 @@ jobs: - uses: actions/setup-python@main with: python-version: '3.9.15' - - run: | + + - name: Install Python dependencies + run: | python -m pip install --upgrade pip - python -m pip install numpy + python -m pip install numpy xmipp3_installer - uses: Jimver/cuda-toolkit@master if: matrix.cuda.version != 'None' @@ -53,14 +60,13 @@ jobs: - uses: actions/checkout@main with: fetch-depth: 0 - ref: ${{ github.head_ref }} + ref: ${{ steps.extract_branch.outputs.branch }} - - name: Compile Xmipp with all dependencies in branch ${{ github.head_ref }} (if such branch exists, default is devel) + # Variable in the name is only evaluated once the workflow reaches this step + - name: Compile Xmipp with all dependencies in branch ${{ steps.extract_branch.outputs.branch }} (if such branch exists, default is devel) env: - BRANCH_NAME: ${{ github.head_ref }} - SEND_INSTALLATION_STATISTICS: 'OFF' - run: | - ./xmipp all -b $BRANCH_NAME --keep-output || (cat compilation.log && false) + BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} + run: echo "SEND_INSTALLATION_STATISTICS=OFF" > xmipp.conf && ./xmipp all -b $BRANCH_NAME --keep-output || (cat compilation.log && false) - name: Cat run: cat xmipp.conf diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d17fec0a1c..eec07b69c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@main + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + + - name: Install installer package + run: pip install xmipp3_installer - name: Retrieve tag name, release name & changelog run: | @@ -28,9 +34,18 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} custom_tag: ${{ steps.variables.outputs.TAG_NAME }} tag_prefix: '' + + - name: Update major version tag + run: | + VERSION=${{ steps.variables.outputs.TAG_NAME }} + MAJOR=${VERSION%%.*} + git config --global user.name 'GitHub Actions' + git config --global user.email 'action@github.com' + git tag -fa "${MAJOR}" -m 'Update major version tag with $VERSION' + git push origin "${MAJOR}" --force - name: Create a GitHub release - uses: ncipollo/release-action@main + uses: ncipollo/release-action@v1 with: tag: ${{ steps.tag_version.outputs.new_tag }} name: ${{ steps.variables.outputs.RELEASE_NAME }} diff --git a/.sonarcloud.properties b/.sonarcloud.properties index 0b40b9beca..08a407894c 100644 --- a/.sonarcloud.properties +++ b/.sonarcloud.properties @@ -3,7 +3,7 @@ sonar.organization=i2pc # This is the name and version displayed in the SonarCloud UI. sonar.projectVersion=3.0 -sonar.sources=src/xmipp, scripts, installer +sonar.sources=src/xmipp, scripts # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. # NOTE - be careful with excusion rules. If you include formerly excluded folder, all files that include diff --git a/installer/__init__.py b/installer/__init__.py deleted file mode 100644 index 0a10ccb9c2..0000000000 --- a/installer/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -This module allows an intuitive and easy installation of Xmipp. -""" \ No newline at end of file diff --git a/installer/api.py b/installer/api.py deleted file mode 100644 index a766d1874a..0000000000 --- a/installer/api.py +++ /dev/null @@ -1,245 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -Module containing all functions needed for the metric's API request. -""" - -# General imports -import re, hashlib, http.client, json -from typing import Dict, Optional -from urllib.parse import urlparse -import os - -# Self imports -from .cmake import parseCmakeVersions -from .utils import runJob, getCurrentName, isBranchUpToDate, runParallelJobs -from .constants import (API_URL, LOG_FILE, TAIL_LOG_NCHARS, UNKNOWN_VALUE, - XMIPP_VERSIONS, XMIPP, VERSION_KEY, MASTER_BRANCHNAME, VERSION_FILE, CMAKE_PYTHON, - CMAKE_CUDA, CMAKE_MPI, CMAKE_HDF5, CMAKE_JPEG, CMAKE_SQLITE, CMAKE_JAVA, - CMAKE_CMAKE, CMAKE_GCC, CMAKE_GPP) - -def sendApiPOST(retCode: int=0): - """ - ### Sends a POST request to Xmipp's metrics's API. - - #### Params: - - retCode (int): Optional. Return code for the API request. - """ - # Getting JSON data for curl command - bodyParams = __getJSON(retCode=retCode) - # Send API POST request if there were no errors - if bodyParams is not None: - # Define the parameters for the POST request - params = json.dumps(bodyParams) - # Set up the headers - headers = {"Content-type": "application/json"} - parsedUrl = urlparse(API_URL) - # Establish a connection - conn = http.client.HTTPSConnection(parsedUrl.hostname, parsedUrl.port, timeout=6) - try: - # Send the POST request - conn.request("POST", parsedUrl.path, body=params, headers=headers) - # Wait for the response from the server, otherwise could fail in the server - _ = conn.getresponse() - except Exception: - pass - finally: - # Close the connection - conn.close() - - -####################### UTILS FUNCTIONS ####################### -def getOSReleaseName() -> str: - """ - ### This function returns the name of the current system OS release. - - #### Returns: - - (str): OS release name. - """ - # Initializing default release name - releaseName = UNKNOWN_VALUE - - # Text around release name - textBefore = 'PRETTY_NAME="' - textAfter = '"\n' - - # Obtaining os release name - retCode, name = runJob('cat /etc/os-release') - - # Look for release name if command did not fail - if retCode == 0: - # Find release name's line in command output - targetStart = name.find(textBefore) - if targetStart != 1: - # Search end of release name's line - nameEnd = name[targetStart:].find(textAfter) - - # Calculate release name's start index - nameStart = targetStart + len(textBefore) - if nameEnd != -1 and nameStart != nameEnd: - # If everything was correctly found and string is - # not empty, extract release name - releaseName = name[nameStart:nameEnd] - - # Return release name - return releaseName - -def __getJSON(retCode: int=0) -> Optional[Dict]: - """ - ### Creates a JSON with the necessary data for the API POST message. - - #### Params: - - retCode (int): Optional. Return code for the API request. - - #### Return: - - (dict | None): JSON with the required info or None if user id could not be produced. - """ - # Getting user id and checking if it exists - userId = __getUserId() - - # Obtaining variables in parallel - data = parseCmakeVersions(VERSION_FILE) - jsonData = runParallelJobs([ - (getOSReleaseName, ()), - (__getCPUFlags, ()), - (getCurrentName, ()), - (isBranchUpToDate, ()), - (__getLogTail, ()) - ]) - - # If branch is master or there is none, get release name - branchName = XMIPP_VERSIONS[XMIPP][VERSION_KEY] if not jsonData[2] or jsonData[2] == MASTER_BRANCHNAME else jsonData[2] - installedByScipion = bool(os.getenv("SCIPION_SOFTWARE")) - - # Introducing data into a dictionary - return { - "user": { - "userId": userId - }, - "version": { - "os": jsonData[0], - "architecture": jsonData[1], - "cuda": data.get(CMAKE_CUDA), - "cmake": data.get(CMAKE_CMAKE), - "gcc": data.get(CMAKE_GCC), - "gpp": data.get(CMAKE_GPP), - "mpi": data.get(CMAKE_MPI), - "python": data.get(CMAKE_PYTHON), - "sqlite": data.get(CMAKE_SQLITE), - "java": data.get(CMAKE_JAVA), - "hdf5": data.get(CMAKE_HDF5), - "jpeg": data.get(CMAKE_JPEG) - }, - "xmipp": { - "branch": branchName, - "updated": jsonData[3], - "installedByScipion": installedByScipion - }, - "returnCode": retCode, - "logTail": jsonData[4] if retCode else None # Only needs log tail if something went wrong - } - -def __getMACAddress() -> Optional[str]: - """ - ### This function returns a physical MAC address for this machine. It prioritizes ethernet over wireless. - - #### Returns: - - (str | None): MAC address, or None if there were any errors. - """ - # Run command to get network interfaces info - status, output = runJob("ip addr") - - # If command failed, return None to avoid sending POST request - if status != 0: - return - - # Regular expression to match the MAC address and interface names - macRegex = r"link/ether ([0-9a-f:]{17})" - interfaceRegex = r"^\d+: (enp|wlp|eth|ens|eno)\w+" - - # Split the output into lines - lines = output.split('\n') - - # Iterate over the lines looking for MAC address - macAddress = None - for line in lines: - # If this line contains an interface name - if re.match(interfaceRegex, line): - # Extract the interface name - interfaceName = re.match(interfaceRegex, line).group(1) - - # If the interface name starts with 'enp', 'wlp', or 'eth - if interfaceName.startswith(('enp', 'wlp', 'eth', 'ens', 'eno')): - # Extract the MAC address from the next line and exit - macAddress = re.search(macRegex, lines[lines.index(line) + 1]).group(1) - break - - return macAddress - -def __getUserId() -> Optional[str]: - """ - ### This function returns the unique user id for this machine. - - #### Returns: - - (str | None): User id, or None if there were any errors. - """ - # Obtaining user's MAC address - macAddress = __getMACAddress() - - # If no physical MAC address was found, user id cannot be created - if macAddress is None or not macAddress: - return 'Anonymous' - - # Create a new SHA-256 hash object - sha256 = hashlib.sha256() - - # Update the hash object with the bytes of the MAC address - sha256.update(macAddress.encode()) - - # Return hexadecimal representation of the hash - return sha256.hexdigest() - -def __getLogTail() -> Optional[str]: - """ - ### This function returns the last lines of the installation log. - - #### Returns: - - (str | None): Installation log's last lines, or None if there were any errors. - """ - # Obtaining log tail - retCode, output = runJob(f"tail -n {TAIL_LOG_NCHARS} {LOG_FILE}") - - # Return content if it went right - return output if retCode == 0 else None - -def __getCPUFlags() -> str: - """ - ### This function returns a string with the flags provided by lscpu. - """ - returnCode, outputStr = runJob('lscpu | grep Flags') - if returnCode == 0: - flagsCPU = outputStr.replace('Flags:', '').strip() - return flagsCPU - return UNKNOWN_VALUE diff --git a/installer/cmake.py b/installer/cmake.py deleted file mode 100644 index 1bc2fbef6e..0000000000 --- a/installer/cmake.py +++ /dev/null @@ -1,98 +0,0 @@ -# *************************************************************************** -# * Authors: Oier Lauzirika Zarrabeitia (oierlauzi@bizkaia.eu) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -import shutil, os -from typing import Dict, Any, List -from .constants import CMAKE, DEFAULT_CMAKE, INTERNAL_LOGIC_VARS -from .utils import runJob - -def getCMake(config: Dict[str, Any]) -> str: - """ - ### Retrieves information about the CMake package and updates the dictionary accordingly. - - #### Params: - - packages (dict): Dictionary containing package information. - - #### Returns: - - (dict): Param 'packages' with the 'CMAKE' key updated based on the availability of 'cmake'. - """ - cmake = config.get(CMAKE) - return cmake if cmake else shutil.which(DEFAULT_CMAKE) - -def getCMakeVars(config: Dict) -> List[str]: - """ - ### This function converts the variables in the config dictionary into a list as CMake args. - - #### Params: - - configDict (dict): Dictionary to obtain the parameters from. - """ - result = [] - for (key, value) in config.items(): - if key not in INTERNAL_LOGIC_VARS and bool(value): - result.append(f"-D{key}={value}") - return result - -def getCMakeVarsStr(config: Dict) -> str: - """ - ### This function converts the variables in the config dictionary into a string as CMake args. - - #### Params: - - configDict (dict): Dictionary to obtain the parameters from. - """ - return ' '.join(getCMakeVars(config)) - -def checkPackage(package: str, config: Dict[str, Any]) -> bool: - cmake = getCMake(config) - args = [] - args.append(f'-DNAME={package}') - args.append('-DCOMPILER_ID=GNU') - args.append('-DLANGUAGE=C') - args.append('-DMODE=EXIST') - args += getCMakeVars(config) - - cmd = cmake + ' ' + ' '.join(args) - ret, _ = runJob(cmd) - return ret == 0 - -def parseCmakeVersions(path: str) -> Dict[str, Any]: - """ - ### This function parses the file where versions found by CMake have been extracted. - - #### Params: - - path (str): Path to the file containing all versions. - - #### Returns: - - (dict): Dictionary containing all the versions from the file. - """ - result = {} - if not os.path.exists(path): - return result - - with open(path, 'r') as file: - for line in file.readlines(): - if line: - packageLine = line.replace("\n", "").split('=') - value = packageLine[1] if packageLine[1] else None - result[packageLine[0]] = value - - return result diff --git a/installer/config.py b/installer/config.py deleted file mode 100644 index a7d57e14e9..0000000000 --- a/installer/config.py +++ /dev/null @@ -1,172 +0,0 @@ -# *************************************************************************** -# * Authors: Oier Lauzirika Zarrabeitia (oierlauzi@bizkaia.eu) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -import os, re -from typing import Dict, Tuple, Optional -from datetime import datetime -from copy import copy -from .logger import logger, yellow -from .constants import (CONFIG_VARIABLES, CONFIG_DEFAULT_VALUES, TOGGLES, - LOCATIONS, COMPILATION_FLAGS, ON, OFF, CONFIG_FILE) - -__ASSIGNMENT_SEPARATOR = '=' -__COMMENT_ESCAPE = '#' -__LAST_MODIFIED_TEXT = "Config file automatically generated on" - -def __parseConfigLine(lineNumber: int, line: str) -> Optional[Tuple[str, str]]: - """ - ### Reads the given line from the config file and returns the key-value pair as a tuple. - - #### Params: - - lineNumber (int): Line nunber inside the config file. - - line (str): Line to parse. - - #### Returns: - - (tuple(str, str)): Tuple containing the read key-value pair. - """ - # Skip if comments - line = line.split(__COMMENT_ESCAPE, maxsplit=2)[0].strip() - - # Check if empty line - if not line: - return None - - # Try to parse the line - tokens = line.split(__ASSIGNMENT_SEPARATOR, maxsplit=1) - if len(tokens) != 2: - raise RuntimeError(f'Unable to parse line {lineNumber+1}: {line}') - - key = tokens[0].strip() - value = tokens[1].strip() - return key, value - -def __makeConfigLine(key: str, value: str, defaultValue: str) -> str: - """ - ### Composes a config file line given a key-value pair to write. - - #### Params: - - key (int): Name of the variable. - - value (str): Value of the variable found in the config file. - - defaultValue (str): Default value of the variable. - - #### Returns: - - (str): String containing the appropiately formatted key-value pair. - """ - defaultValue = '' if defaultValue is None else defaultValue - value = defaultValue if value is None else value - return key + __ASSIGNMENT_SEPARATOR + value - -def readConfig(path: str) -> Dict[str, str]: - """ - ### Reads the config file and returns a dictionary with all the parsed variables. - - #### Params: - - path (str): Path to the config file. - - #### Returns: - - (dict): Dictionary containing all the variables found in the config file. - """ - result = dict() - - with open(path, 'r') as configFile: - for i, line in enumerate(configFile): - try: - keyval = __parseConfigLine(i, line) - except RuntimeError as rte: - warningStr = f"WARNING: There was an error parsing {CONFIG_FILE} file: {rte}\n" - warningStr += "Contents of config file won't be read, default values will be used instead.\n" - warningStr += "You can create a new file template from scratch running './xmipp config -o'." - logger(yellow(warningStr), forceConsoleOutput=True) - return CONFIG_DEFAULT_VALUES - if keyval is not None: - key, value = keyval - result[key] = value - - return result - -def writeConfig(path: str, configDict: Dict=None): - """ - ### Writes a template config file with given variables, leaving the rest with default values. - - #### Params: - - path (str): Path to the config file. - - configDict (dict): Optional. Dictionary containig already existing variables. - """ - variables = copy(configDict) if configDict else {} - lines = [] - with open(path, 'w') as configFile: - lines.append("##### TOGGLE SECTION #####\n") - lines.append(f"# Activate or deactivate this features using values {ON}/{OFF}\n") - for toggle in CONFIG_VARIABLES[TOGGLES]: - lines.append(__makeConfigLine(toggle, variables.get(toggle), CONFIG_DEFAULT_VALUES[toggle]) + '\n') - variables.pop(toggle, None) - - lines.append("\n##### PACKAGE HOME SECTION #####\n") - lines.append("# Use this variables to use custom installation paths for the required packages.\n") - lines.append("# If left empty, CMake will search for those packages within your system.\n") - for location in CONFIG_VARIABLES[LOCATIONS]: - lines.append(__makeConfigLine(location, variables.get(location), CONFIG_DEFAULT_VALUES[location]) + '\n') - variables.pop(location, None) - - lines.append("\n##### COMPILATION FLAGS #####\n") - lines.append("# We recommend not modifying this variables unless you know what you are doing.\n") - for flag in CONFIG_VARIABLES[COMPILATION_FLAGS]: - lines.append(__makeConfigLine(flag, variables.get(flag), CONFIG_DEFAULT_VALUES[flag]) + '\n') - variables.pop(flag, None) - - # If there are extra unkown flags, add them in a different section - if variables: - lines.append("\n##### UNKNOWN VARIABLES #####\n") - lines.append("# This variables were not expected, but are kept here in case they might be needed.\n") - for variable in variables.keys(): - lines.append(__makeConfigLine(variable, variables[variable], '') + '\n') - - lines.append(f"\n# {__LAST_MODIFIED_TEXT} {datetime.today()}\n") - configFile.writelines(lines) - -def getConfigDate(path: str) -> str: - """ - ### This function obtains from the config file the date of its last modification. - - #### Params: - - path (str): Path to the config file. - - #### Returns: - - (str): Date formatted in dd/mm/yyyy. - """ - if not os.path.exists(path): - return '' - - # Extract line with date, and date from such line - dateStr = '' - with open(path, 'r') as configFile: - for line in configFile: - if __LAST_MODIFIED_TEXT in line: - match = re.search(r'\d{4}-\d{2}-\d{2}', line) - if match: - dateStr = match.group() - - if dateStr: - dateStr = datetime.strptime(dateStr, '%Y-%m-%d').strftime('%d/%m/%Y') - - return dateStr diff --git a/installer/constants/__init__.py b/installer/constants/__init__.py deleted file mode 100644 index aef174bc63..0000000000 --- a/installer/constants/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -Module containing all constants needed for the installation of Xmipp. -""" - -from .main import * -from .errors import * -from .parser import * -from .versions import * -from .config import * -from .cmake import * diff --git a/installer/constants/cmake.py b/installer/constants/cmake.py deleted file mode 100644 index 3fff991165..0000000000 --- a/installer/constants/cmake.py +++ /dev/null @@ -1,49 +0,0 @@ -# *************************************************************************** -# * Authors: Martín Salinas (martin.salinas@cnb.csic.es) -# * Oier Lauzirika Zarrabeitia (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -from .config import CUDA, MPI, MATLAB, LINK_SCIPION, CC, CXX, CUDA_COMPILER - -DEFAULT_CMAKE = 'cmake' - -# CMake cache file variables to look for -XMIPP_USE_CUDA=CUDA -XMIPP_USE_MPI=MPI -XMIPP_USE_MATLAB=MATLAB -XMIPP_LINK_TO_SCIPION=LINK_SCIPION -CMAKE_BUILD_TYPE='CMAKE_BUILD_TYPE' -CMAKE_C_COMPILER=CC -CMAKE_CXX_COMPILER=CXX -CMAKE_CUDA_COMPILER=CUDA_COMPILER - -# CMake saved version variables -CMAKE_PYTHON = 'Python3' -CMAKE_CUDA = 'CUDA' -CMAKE_MPI = 'MPI' -CMAKE_HDF5 = 'HDF5' -CMAKE_JPEG = 'JPEG' -CMAKE_SQLITE = 'SQLite3' -CMAKE_JAVA = 'Java' -CMAKE_CMAKE = 'CMake' -CMAKE_GCC = 'CC' -CMAKE_GPP = 'CXX' diff --git a/installer/constants/config.py b/installer/constants/config.py deleted file mode 100644 index 2b035523ee..0000000000 --- a/installer/constants/config.py +++ /dev/null @@ -1,137 +0,0 @@ -# *************************************************************************** -# * Authors: Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -import os -from typing import Optional - -from .main import INSTALL_PATH - -# Variable names -SEND_INSTALLATION_STATISTICS = 'SEND_INSTALLATION_STATISTICS' -__SEND_INSTALLATION_STATISTICS_ENV = 'SEND_INSTALLATION_STATISTICS' -CMAKE = 'CMAKE' -CC = 'CMAKE_C_COMPILER' -CXX = 'CMAKE_CXX_COMPILER' -__CC_FLAGS = 'CMAKE_C_FLAGS' -__CXX_FLAGS = 'CMAKE_CXX_FLAGS' -CMAKE_INSTALL_PREFIX = 'CMAKE_INSTALL_PREFIX' -CUDA = 'XMIPP_USE_CUDA' -CUDA_COMPILER = 'CMAKE_CUDA_COMPILER' -MPI = 'XMIPP_USE_MPI' -__PREFIX_PATH = 'CMAKE_PREFIX_PATH' -__MPI_HOME = 'MPI_HOME' -__PYTHON_HOME = 'Python3_ROOT_DIR' -__FFTW_HOME = 'FFTW_ROOT' -__TIFF_HOME = 'TIFF_ROOT' -__HDF5_HOME = 'HDF5_ROOT' -__JPEG_HOME = 'JPEG_ROOT' -__SQLITE_HOME = 'SQLite_ROOT' -__CUDA_CXX = 'CMAKE_CUDA_HOST_COMPILER' -MATLAB = 'XMIPP_USE_MATLAB' -LINK_SCIPION = 'XMIPP_LINK_TO_SCIPION' -__BUILD_TESTING = 'BUILD_TESTING' -__SKIP_RPATH='CMAKE_SKIP_RPATH' - -# This is not used in cmake -__CONDA_PREFIX = 'CONDA_PREFIX' -__XMIPP_CUDA_BIN = 'XMIPP_CUDA_BIN' -__DEFAULT_CUDA_BIN = '/usr/local/cuda/bin' -__NVCC_EXE = 'nvcc' -__TUNE_FLAG='-mtune=native' - -# Config file variable structure -TOGGLES = 'toggles' -LOCATIONS = 'locations' -COMPILATION_FLAGS = 'flags' -CONFIG_VARIABLES = { - TOGGLES: [ - __SEND_INSTALLATION_STATISTICS_ENV, CUDA, MPI, MATLAB, LINK_SCIPION, __BUILD_TESTING, __SKIP_RPATH - ], - LOCATIONS: [ - CMAKE, CC, CXX, CMAKE_INSTALL_PREFIX, __PREFIX_PATH, __MPI_HOME, - CUDA_COMPILER, __PYTHON_HOME, __FFTW_HOME, __TIFF_HOME, - __HDF5_HOME, __JPEG_HOME, __SQLITE_HOME, __CUDA_CXX - ], - COMPILATION_FLAGS: [__CC_FLAGS, __CXX_FLAGS] -} - -def __getPrefixPath() -> Optional[str]: - """ - ### This function returns the path for the current Conda enviroment. - - #### Returns: - - (str | None): Path for current Conda enviroment. - """ - return os.environ.get(__CONDA_PREFIX) - -def __getCudaCompiler() -> Optional[str]: - """ - ### This function returns the path for the CUDA compiller - - #### Returns: - - (str | None): Path for the NVCC executable - """ - nvcc = os.environ.get(__XMIPP_CUDA_BIN) - - if nvcc is None and os.path.exists(__DEFAULT_CUDA_BIN): - nvcc = __DEFAULT_CUDA_BIN - - if nvcc is not None: - nvcc = os.path.join(nvcc, __NVCC_EXE) - - return nvcc - -def __getSendStatistics(): - return os.environ.get(__SEND_INSTALLATION_STATISTICS_ENV, ON) - - -ON = 'ON' -OFF = 'OFF' -CONFIG_DEFAULT_VALUES = { - __SEND_INSTALLATION_STATISTICS_ENV: __getSendStatistics(), - CMAKE: None, - CUDA: ON, - MPI: ON, - CC: None, - CXX: None, - CMAKE_INSTALL_PREFIX: INSTALL_PATH, - __CC_FLAGS: __TUNE_FLAG, - __CXX_FLAGS: __TUNE_FLAG, - CUDA_COMPILER: __getCudaCompiler(), - __PREFIX_PATH: __getPrefixPath(), - __MPI_HOME: None, - __PYTHON_HOME: None, - __FFTW_HOME: None, - __TIFF_HOME: None, - __HDF5_HOME: None, - __JPEG_HOME: None, - __SQLITE_HOME: None, - __CUDA_CXX: None, - MATLAB: ON, - LINK_SCIPION: ON, - __BUILD_TESTING: ON, - __SKIP_RPATH: ON -} - -# Do not pass this variables to CMake, only for installer logic -INTERNAL_LOGIC_VARS = [__SEND_INSTALLATION_STATISTICS_ENV, CMAKE] diff --git a/installer/constants/errors.py b/installer/constants/errors.py deleted file mode 100644 index 6b30961ddc..0000000000 --- a/installer/constants/errors.py +++ /dev/null @@ -1,56 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -Submodule containing all constants needed for handling errors during Xmipp's installation. -""" - -from .main import CMAKE_INSTALL_DOCS_URL, LOG_FILE, CONFIG_FILE - -# Error codes -INTERRUPTED_ERROR = -1 -OK = 0 -UNKOW_ERROR = 1 -SOURCE_CLONE_ERROR = 2 -CMAKE_ERROR = 3 -CMAKE_CONFIGURE_ERROR = 4 -CMAKE_COMPILE_ERROR = 5 -CMAKE_INSTALL_ERROR = 6 -IO_ERROR = 7 -ENVIROMENT_ERROR = 8 - -# Error messages -__CHECK_LOG_MESSAGE = f'Check the inside file \'{LOG_FILE}\'.' -ERROR_CODE = { - INTERRUPTED_ERROR: ['Process was interrupted by the user.', ''], - UNKOW_ERROR: ['', ''], - SOURCE_CLONE_ERROR: ['Error cloning xmipp repository with git.', 'Please review the internet connection and the git package.'], - CMAKE_ERROR: ['There was an error with CMake.', f'Please install it by following the instructions at {CMAKE_INSTALL_DOCS_URL}'], - CMAKE_CONFIGURE_ERROR: ['Error configuring with CMake.', __CHECK_LOG_MESSAGE], - CMAKE_COMPILE_ERROR: ['Error compiling with CMake.', __CHECK_LOG_MESSAGE], - CMAKE_INSTALL_ERROR: ['Error installing with CMake.', __CHECK_LOG_MESSAGE], - IO_ERROR: ['Input/output error.', 'This error can be caused by the installer not being able to read/write/create/delete a file. Check your permissions on this directory.'], -ENVIROMENT_ERROR: ['XMIPP_SRC is not in the enviroment. To run the tests you need to run: source dist/xmipp.bashrc', ''] - -} diff --git a/installer/constants/main.py b/installer/constants/main.py deleted file mode 100644 index 5dda508381..0000000000 --- a/installer/constants/main.py +++ /dev/null @@ -1,82 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -Submodule containing all the general constants needed for Xmipp's installation. -""" - -import os - -XMIPP = 'xmipp' -XMIPP_CORE = 'xmippCore' -XMIPP_VIZ = 'xmippViz' -XMIPP_PLUGIN = 'scipion-em-xmipp' - -# Formatting characters -UP = "\x1B[1A\r" -REMOVE_LINE = '\033[K' -BOLD = "\033[1m" -BLUE = "\033[34m" -RED = "\033[91m" -GREEN = "\033[92m" -YELLOW = "\033[93m" -END_FORMAT = "\033[0m" -FORMATTING_CHARACTERS = [UP, REMOVE_LINE, BOLD, BLUE, RED, GREEN, YELLOW, END_FORMAT] - -__BASE_GITHUB_URL = 'https://github.com' -__ORGANIZATION_NAME = 'I2PC' -REPOSITORIES = { - XMIPP: [os.path.join(__BASE_GITHUB_URL, __ORGANIZATION_NAME, XMIPP), None], - XMIPP_CORE: [os.path.join(__BASE_GITHUB_URL, __ORGANIZATION_NAME, XMIPP_CORE), None], - XMIPP_VIZ: [os.path.join(__BASE_GITHUB_URL, __ORGANIZATION_NAME, XMIPP_VIZ), None], - XMIPP_PLUGIN: [os.path.join(__BASE_GITHUB_URL, __ORGANIZATION_NAME, XMIPP_PLUGIN), None], -} -XMIPP_SOURCES = [XMIPP_CORE, XMIPP_VIZ, XMIPP_PLUGIN] -SOURCES_PATH = "src" -BUILD_PATH = "build" -INSTALL_PATH = "dist" -BUILD_TYPE = "Release" -CMAKE_CACHE_PATH = os.path.join(BUILD_PATH, 'CMakeCache.txt') - -# Internal URLs -API_URL = 'https://xmipp.i2pc.es/api/attempts/' -DOCUMENTATION_URL = 'https://i2pc.github.io/docs/' - -# External URLs -CMAKE_INSTALL_DOCS_URL = 'https://i2pc.github.io/docs/Installation/Requirements/index.html#requirements' -MODELS_URL = "http://scipion.cnb.csic.es/downloads/scipion/software/em" -SCIPION_FILES_REMOTE_PATH = "scipionfiles/downloads/scipion/software/em" -SCIPION_TESTS_URLS = "http://scipion.cnb.csic.es/downloads/scipion/data/tests" -SCIPION_SOFTWARE_EM = "scipionfiles/downloads/scipion/software/em" - -# Other variables -TAB_SIZE = 4 -SECTION_MESSAGE_LEN = 60 -TAG_BRANCH_NAME = 'v3.25.06.0-Rhea' - -# File names -CONFIG_FILE = 'xmipp.conf' -LOG_FILE = 'compilation.log' -VERSION_FILE = 'build/versions.txt' -TAIL_LOG_NCHARS = 300 diff --git a/installer/constants/parser.py b/installer/constants/parser.py deleted file mode 100644 index b9f63c3f94..0000000000 --- a/installer/constants/parser.py +++ /dev/null @@ -1,252 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -Submodule containing all constants needed for the argument parsing part of Xmipp's installation. -""" -from .main import INSTALL_PATH - -# Other variables -COMMON_USAGE_HELP_MESSAGE = 'Run \"./xmipp -h\" for usage help.' -DEFAULT_BUILD_DIR = 'build' -DEFAULT_MODELS_DIR = INSTALL_PATH - -# Mode list (alphabetical order) -MODE_ADD_MODEL = 'addModel' -MODE_ALL = 'all' -MODE_CLEAN_ALL = 'cleanAll' -MODE_CLEAN_BIN = 'cleanBin' -MODE_COMPILE_AND_INSTALL = 'compileAndInstall' -MODE_CONFIG_BUILD = 'configBuild' -MODE_CONFIG = 'config' -MODE_GET_MODELS = 'getModels' -MODE_GET_SOURCES = 'getSources' -MODE_GIT = 'git' -MODE_TEST = 'test' -MODE_VERSION = 'version' - -# Modes with help message -MODES = { - 'General': { - MODE_VERSION: ['Returns the version information. Add \'--short\' to print only the version number.'], - MODE_COMPILE_AND_INSTALL: ['Compiles and installs Xmipp based on already obtained sources.'], - MODE_ALL: [f'Default param. Runs {MODE_CONFIG}, {MODE_CONFIG_BUILD}, and {MODE_COMPILE_AND_INSTALL}.'], - MODE_CONFIG_BUILD: ['Configures the project with CMake.'] - }, - 'Config': { - MODE_CONFIG: ['Generates a config file template with default values.'], - }, - 'Downloads': { - MODE_GET_MODELS: [f'Download the DeepLearning Models at dir/models ({DEFAULT_MODELS_DIR} by default).'], - MODE_GET_SOURCES: ['Clone all Xmipp\'s sources.'] - }, - 'Clean': { - MODE_CLEAN_BIN: ['Removes all compiled binaries.'], - MODE_CLEAN_ALL: ['Removes all compiled binaries and sources, leaves the repository as if freshly cloned (without pulling).'] - }, - 'Test': { - MODE_TEST: ['Runs a given test.'] - }, - 'Developers': { - MODE_GIT: ['Runs the given git action for all source repositories.'], - MODE_ADD_MODEL: [ - "Takes a DeepLearning model from the modelPath, makes a tgz of it and uploads the .tgz according to the .", - "This mode is used to upload a model folder to the Scipion/Xmipp server.", - "Usually the model folder contains big files used to fed deep learning procedures" - "with pretrained data. All the models stored in the server will be downloads" - "using the 'get_models' mode or during the compilation/installation time" - "or with scipion3 installb deepLearningToolkit modelsPath must be the absolute path.", - "", - "Usage: -> ./xmipp addModel [--update]", - "Steps: 0. modelName = basename(modelsPath) <- Please, check the folders name!", - " 1. Packing in 'xmipp_model_modelName.tgz'", - " 2. Check if that model already exists (use --update to override an existing model)", - " 3. Upload the model to the server.", - " 4. Update the MANIFEST file.", - "", - "The model name will be the folder name in ", - "Must have write permisions to such machine." - ] - } -} - -# Definition of all params found in the -SHORT_VERSION = 'short' -LONG_VERSION = 'long' -DESCRIPTION = 'description' -# Possible param list -PARAM_SHORT = 'short' -PARAM_JOBS = 'jobs' -PARAM_BRANCH = 'branch' -PARAM_PRODUCTION = 'production' -PARAM_MODELS_DIRECTORY = 'models-directory' -PARAM_TEST_NAME = 'test-name' -PARAM_SHOW_TESTS = 'show-tests' -PARAM_TEST_PRO = 'allPrograms' -PARAM_TEST_FUNC = 'allFuncs' -PARAM_GIT_COMMAND = 'git-command' -PARAM_LOGIN = 'login' -PARAM_MODEL_PATH = 'model-path' -PARAM_UPDATE = 'update' -PARAM_OVERWRITE = 'overwrite' -PARAM_KEEP_OUTPUT = "keep-output" - -PARAMS = { - PARAM_SHORT: { - LONG_VERSION: "--short", - DESCRIPTION: "If set, only version number is shown." - }, - PARAM_JOBS: { - SHORT_VERSION: "-j", - LONG_VERSION: "--jobs", - DESCRIPTION: "Number of jobs. Defaults to all available." - }, - PARAM_BRANCH: { - SHORT_VERSION: "-b", - LONG_VERSION: "--branch", - DESCRIPTION: "Branch for the source repositories." - }, - PARAM_PRODUCTION: { - LONG_VERSION: "--production", - DESCRIPTION: "Flag to identify a production compilation (avoid the download of the plugin)." - }, - PARAM_MODELS_DIRECTORY: { - SHORT_VERSION: "-d", - LONG_VERSION: "--directory", - DESCRIPTION: f"Directory where models will be saved. Default is \"{DEFAULT_MODELS_DIR}\"." - }, - PARAM_TEST_NAME: { - SHORT_VERSION: "testName", - DESCRIPTION: "Run certain test. If combined with --show, greps the test name from the test list." - }, - PARAM_TEST_PRO: { - LONG_VERSION: "--allPrograms", - DESCRIPTION: "If set, all test available will be run." - }, - PARAM_TEST_FUNC: { - LONG_VERSION: "--allFuncs", - DESCRIPTION: "If set, all function test available will be run." - }, - PARAM_SHOW_TESTS: { - LONG_VERSION: "--show", - DESCRIPTION: "Shows the tests available and how to invoke those." - }, - PARAM_GIT_COMMAND: { - SHORT_VERSION: "command", - DESCRIPTION: "Git command to run on all source repositories." - }, - PARAM_LOGIN: { - SHORT_VERSION: "login", - DESCRIPTION: "Login (usr@server) for Nolan machine to upload the model with. Must have write permisions to such machine." - }, - PARAM_MODEL_PATH: { - SHORT_VERSION: "modelPath", - DESCRIPTION: "Path to the model to upload to Nolan." - }, - PARAM_UPDATE: { - LONG_VERSION: "--update", - DESCRIPTION: "Flag to update an existing model" - }, - PARAM_OVERWRITE: { - SHORT_VERSION: "-o", - LONG_VERSION: "--overwrite", - DESCRIPTION: "If set, current config file will be overwritten with a new one." - }, - PARAM_KEEP_OUTPUT: { - LONG_VERSION: "--keep-output", - DESCRIPTION: "If set, output sent through the terminal won't substitute lines, looking more like the log." - } -} - -# Arguments of each mode, sorted by group -MODE_ARGS = { - MODE_VERSION: [PARAM_SHORT], - MODE_COMPILE_AND_INSTALL: [PARAM_JOBS, PARAM_BRANCH, PARAM_KEEP_OUTPUT], - MODE_ALL: [PARAM_JOBS, PARAM_BRANCH, PARAM_KEEP_OUTPUT, PARAM_PRODUCTION], - MODE_CONFIG_BUILD: [PARAM_KEEP_OUTPUT], - MODE_CONFIG: [PARAM_OVERWRITE], - MODE_GET_MODELS: [PARAM_MODELS_DIRECTORY], - MODE_GET_SOURCES: [PARAM_BRANCH, PARAM_KEEP_OUTPUT], - MODE_CLEAN_BIN: [], - MODE_CLEAN_ALL: [], - MODE_TEST: [PARAM_TEST_NAME, PARAM_SHOW_TESTS, PARAM_TEST_FUNC, PARAM_TEST_PRO], - MODE_GIT: [PARAM_GIT_COMMAND], - MODE_ADD_MODEL: [PARAM_LOGIN, PARAM_MODEL_PATH, PARAM_UPDATE] -} - -# Examples for the help message of each mode -MODE_EXAMPLES = { - MODE_VERSION: [ - f'./xmipp {MODE_VERSION}', - f'./xmipp {MODE_VERSION} {PARAMS[PARAM_SHORT][LONG_VERSION]}', - ], - MODE_COMPILE_AND_INSTALL: [ - f'./xmipp {MODE_COMPILE_AND_INSTALL}', - f'./xmipp {MODE_COMPILE_AND_INSTALL} {PARAMS[PARAM_JOBS][SHORT_VERSION]} 20', - f'./xmipp {MODE_COMPILE_AND_INSTALL} {PARAMS[PARAM_BRANCH][SHORT_VERSION]} devel', - f'./xmipp {MODE_COMPILE_AND_INSTALL} {PARAMS[PARAM_JOBS][SHORT_VERSION]} ' - f'20 {PARAMS[PARAM_BRANCH][SHORT_VERSION]} devel' - ], - MODE_ALL: [ - './xmipp', - f'./xmipp {MODE_ALL}', - f'./xmipp {PARAMS[PARAM_JOBS][SHORT_VERSION]} 20', - f'./xmipp {PARAMS[PARAM_PRODUCTION][LONG_VERSION]}', - f'./xmipp {PARAMS[PARAM_BRANCH][SHORT_VERSION]} True', - f'./xmipp {MODE_ALL} {PARAMS[PARAM_JOBS][SHORT_VERSION]} 20 ' - f'{PARAMS[PARAM_BRANCH][SHORT_VERSION]} devel' - ], - MODE_CONFIG_BUILD: [], - MODE_CONFIG: [ - f'./xmipp {MODE_CONFIG} {PARAMS[PARAM_OVERWRITE][LONG_VERSION]}' - ], - MODE_GET_MODELS: [ - f'./xmipp {MODE_GET_MODELS}', - f'./xmipp {MODE_GET_MODELS} -directory {PARAMS[PARAM_MODELS_DIRECTORY][SHORT_VERSION]} /path/to/my/model/directory', - f'./xmipp {MODE_GET_MODELS} -d {PARAMS[PARAM_MODELS_DIRECTORY][SHORT_VERSION]} /path/to/my/model/directory' - - ], - MODE_GET_SOURCES: [ - f'./xmipp {MODE_GET_SOURCES}' - f'./xmipp {MODE_GET_SOURCES} {PARAMS[PARAM_BRANCH][SHORT_VERSION]} devel' - ], - MODE_CLEAN_BIN: [], - MODE_CLEAN_ALL: [], - MODE_TEST: [ - f'./xmipp {MODE_TEST} xmipp_sample_test', - f'./xmipp {MODE_TEST} {PARAMS[PARAM_SHOW_TESTS][LONG_VERSION]}', - f'./xmipp {MODE_TEST} {PARAMS[PARAM_TEST_FUNC][LONG_VERSION]}', - f'./xmipp {MODE_TEST} {PARAMS[PARAM_TEST_PRO][LONG_VERSION]}', - - ], - MODE_GIT: [ - f'./xmipp {MODE_GIT} pull', - f'./xmipp {MODE_GIT} checkout devel' - ], - MODE_ADD_MODEL: [ - f'./xmipp {MODE_ADD_MODEL} myuser@127.0.0.1 /home/myuser/mymodel', - f'./xmipp {MODE_ADD_MODEL} myuser@127.0.0.1 /home/myuser/mymodel --update' - - ] -} diff --git a/installer/constants/versions.py b/installer/constants/versions.py deleted file mode 100644 index 7d2a1422a8..0000000000 --- a/installer/constants/versions.py +++ /dev/null @@ -1,61 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' - -""" -Submodule containing all version info required for Xmipp's installation process. -""" - -from .main import XMIPP, XMIPP_CORE, XMIPP_VIZ, XMIPP_PLUGIN - -# Xmipp's current versions -LATEST_RELEASE_NUMBER = '3.25.06.0' -LATEST_RELEASE_NAME = 'v3.25.06.0-Rhea' -RELEASE_DATE = '24/06/2025' -##################################### -DEVEL_BRANCHNAME = 'devel' -MASTER_BRANCHNAME = 'master' - -VERSION_KEY = 'version' -VERNAME_KEY = 'vername' -XMIPP_VERSIONS = { - XMIPP: { - VERSION_KEY: LATEST_RELEASE_NUMBER, - VERNAME_KEY: LATEST_RELEASE_NAME - }, - XMIPP_CORE: { - VERSION_KEY: LATEST_RELEASE_NUMBER, - VERNAME_KEY: LATEST_RELEASE_NAME - }, - XMIPP_VIZ: { - VERSION_KEY: LATEST_RELEASE_NUMBER, - VERNAME_KEY: LATEST_RELEASE_NAME - }, - XMIPP_PLUGIN: { - VERSION_KEY: LATEST_RELEASE_NUMBER, - VERNAME_KEY: LATEST_RELEASE_NAME - } -} -##################################### - -# Other variables -UNKNOWN_VALUE = 'Unknown' diff --git a/installer/logger.py b/installer/logger.py deleted file mode 100644 index 28edf97eac..0000000000 --- a/installer/logger.py +++ /dev/null @@ -1,206 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * Martín Salinas (martin.salinas@cnb.csic.es) -# * Oier Lauzirika Zarrabeitia (olauzirika@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -Provides a global logger. -""" - -# General imports -import shutil, math - -# Installer imports -from .constants import (ERROR_CODE, DOCUMENTATION_URL, UP, REMOVE_LINE, - BOLD, BLUE, RED, GREEN, YELLOW, END_FORMAT, FORMATTING_CHARACTERS) - -####################### TEXT MODE ####################### -def green(text: str) -> str: - """ - ### This function returns the given text formatted in green color. - - #### Params: - - text (str): Text to format. - - #### Returns: - - (str): Text formatted in green color. - """ - return f"{GREEN}{text}{END_FORMAT}" - -def yellow(text: str) -> str: - """ - ### This function returns the given text formatted in yellow color. - - #### Params: - - text (str): Text to format. - - #### Returns: - - (str): Text formatted in yellow color. - """ - return f"{YELLOW}{text}{END_FORMAT}" - -def red(text: str) -> str: - """ - ### This function returns the given text formatted in red color. - - #### Params: - - text (str): Text to format. - - #### Returns: - - (str): Text formatted in red color. - """ - return f"{RED}{text}{END_FORMAT}" - -def blue(text: str) -> str: - """ - ### This function returns the given text formatted in blue color. - - #### Params: - - text (str): Text to format. - - #### Returns: - - (str): Text formatted in blue color. - """ - return f"{BLUE}{text}{END_FORMAT}" - -def bold(text: str) -> str: - """ - ### This function returns the given text formatted in bold. - - #### Params: - - text (str): Text to format. - - #### Returns: - - (str): Text formatted in bold. - """ - return f"{BOLD}{text}{END_FORMAT}" - -def removeNonPrintable(text: str) -> str: - """ - ### This function returns the given text without non printable characters. - - #### Params: - - text (str): Text to remove format. - - #### Returns: - - (str): Text without format. - """ - for formattingChar in FORMATTING_CHARACTERS: - text = text.replace(formattingChar, "") - return text - -class Logger: - """ - ### Logger class for keeping track of installation messages. - """ - - def __init__(self, outputToConsole: bool = False): - """ - ### Constructor. - - #### Params: - - ouputToConsoloe (bool): Print messages to console. - """ - self.__logFile = None - self.__outputToConsole = outputToConsole - self.__lenLastPrintedElem = 0 - self.__allowSubstitution = True - - def startLogFile(self, logPath: str): - """ - ### Initiates the log file. - - #### Params: - - logPath (str): Path to the log file. - """ - self.__logFile = open(logPath, 'w') - - def setConsoleOutput(self, outputToConsole: bool): - """ - ### Modifies console output behaviour. - - #### Params: - - ouputToConsoloe (str): Enable printing messages to console. - """ - self.__outputToConsole = outputToConsole - - def setAllowSubstitution(self, allowSubstitution: bool): - """ - ### Modifies console output behaviour, allowing or disallowing substitutions. - - #### Params: - - allowSubstitution (bool): If False, console outputs won't be substituted. - """ - self.__allowSubstitution = allowSubstitution - - - def __call__(self, text: str, forceConsoleOutput: bool = False, substitute: bool = False): - """ - ### Log a message. - - #### Params: - - text (str): Message to be logged. Supports fancy formatting. - - forceConsoleOutput (bool): Optional. If True, text is also printed through terminal. - - substitute (bool): Optional. If True, previous line is substituted with new text. Only used when forceConsoleOutput = True. - """ - if self.__logFile is not None: - print(removeNonPrintable(text), file=self.__logFile, flush=True) - - if self.__outputToConsole or forceConsoleOutput: - # Calculate number of lines to substitute if substitution was requested - substitutionStr = ''.join([f'{UP}{REMOVE_LINE}' for _ in range(self.__getNLastLines())]) - text = f"{substitutionStr}{text}" if self.__allowSubstitution and substitute else text - print(text, flush=True) - # Store length of printed string for next substitution calculation - self.__lenLastPrintedElem = len(removeNonPrintable(text)) - - def logError(self, errorMsg: str, retCode: int=1, addPortalLink: bool=True): - """ - ### This function prints an error message. - - #### Params: - - errorMsg (str): Error message to show. - - retCode (int): Optional. Return code to end the exection with. - - addPortalLink (bool): If True, a message linking the documentation portal is shown. - """ - errorStr = errorMsg + '\n\n' - errorStr += f'Error {retCode}: {ERROR_CODE[retCode][0]}' - errorStr += f"\n{ERROR_CODE[retCode][1]} " if ERROR_CODE[retCode][1] else '' - if addPortalLink: - errorStr += f'\nMore details on the Xmipp documentation portal: {DOCUMENTATION_URL}' - - self.__call__(red(errorStr), forceConsoleOutput=True) - - def __getNLastLines(self) -> int: - """ - ### This function returns the number of lines of the terminal the last print occupied. - - #### Returns: - - (int): Number of lines of the last print. - """ - return math.ceil(self.__lenLastPrintedElem / shutil.get_terminal_size().columns) - -""" -### Global logger. -""" -logger = Logger() diff --git a/installer/main.py b/installer/main.py deleted file mode 100644 index 42fae3139a..0000000000 --- a/installer/main.py +++ /dev/null @@ -1,348 +0,0 @@ -# *************************************************************************** -# * Authors: Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -This module contains the necessary functions to run most installer commands. -""" - -# General imports -import os, sys -from typing import Tuple, Optional - -# Module imports -from .utils import runJob, getCurrentBranch, isProductionMode, getCurrentName -from .logger import logger, yellow, green, bold -from .constants import (REPOSITORIES, XMIPP_SOURCES, SOURCES_PATH, MASTER_BRANCHNAME, - SOURCE_CLONE_ERROR, TAG_BRANCH_NAME, INTERRUPTED_ERROR, VERSION_FILE, RELEASE_DATE, - XMIPP_VERSIONS, XMIPP, VERSION_KEY, SECTION_MESSAGE_LEN, VERNAME_KEY, MODE_GET_SOURCES, - MODE_CONFIG_BUILD, CONFIG_FILE, XMIPP_PLUGIN) -from .api import sendApiPOST, getOSReleaseName -from .cmake import parseCmakeVersions -from .config import getConfigDate - -####################### COMMAND FUNCTIONS ####################### -def getSources(branch: str=None, production: bool=False): - """ - ### This function fetches the sources needed for Xmipp to compile. - - #### Params: - - branch (str): Optional. Branch to clone the sources from. - """ - # Clone or download internal sources - logger(getSectionMessage("Getting Xmipp sources"), forceConsoleOutput=True) - for source in XMIPP_SOURCES: - if source == XMIPP_PLUGIN and production: - pass - else: - logger(f"Cloning {source}...", forceConsoleOutput=True) - retCode, output = __cloneSourceRepo(REPOSITORIES[source][0], path=SOURCES_PATH, branch=branch) - message = output if retCode else '' - handleRetCode(retCode, predefinedErrorCode=SOURCE_CLONE_ERROR, message=message, sendAPI=False) - -def exitXmipp(retCode: int=0): - """ - ### This function exits Xmipp with the given return code, processing it as a success or an error. - - #### Params: - - retCode (int): Optional. Error code. - """ - # End execution - sys.exit(retCode) - -def handleRetCode(realRetCode: int, predefinedErrorCode: int=0, message: str='', sendAPI: bool=True): - """ - ### This function checks the given return code and handles the appropiate actions. - - #### Params: - - realRetCode (int): Real return code of the called function. - - predefinedErrorCode (int): Optional. Predefined error code for the caller code block in case of error. - - message (str): Optional. Message that will be displayed if there is an error th - - sendAPI (bool): Optional. If True, API message will be sent. - """ - if realRetCode: - resultCode = __getPredefinedError(realRetCode=realRetCode, desiredRetCode=predefinedErrorCode) - message = message if resultCode != realRetCode else '' - logger.logError(message, retCode=resultCode, addPortalLink=resultCode != realRetCode) - - if sendAPI and resultCode != INTERRUPTED_ERROR: - sendApiPOST(resultCode) - exitXmipp(retCode=resultCode) - else: - if message: - logger(message) - __logDoneMessage() - logger("", forceConsoleOutput=True) - -def getSuccessMessage() -> str: - """ - ### This function returns the message shown when Xmipp is compiled successfully. - - #### Returms: - - (str): Success message. - """ - # Getting release name - branchName = getCurrentName() - releaseName = branchName - if branchName is None or branchName == MASTER_BRANCHNAME: - releaseName = XMIPP_VERSIONS[XMIPP][VERSION_KEY] - - # Creating message line - releaseMessage = 'Xmipp {} has been successfully installed, enjoy it!'.format(releaseName) - releaseMessageWrapper = '* *' - messageLine = releaseMessageWrapper[:int(len(releaseMessageWrapper)/2)] - messageLine += green(f'Xmipp {releaseName} has been successfully installed, enjoy it!') - messageLine += releaseMessageWrapper[int(len(releaseMessageWrapper)/2):] - - # Creating box around message line - totalLen = len(releaseMessage) + len(releaseMessageWrapper) - topBottomBorder = ''.join(['*' for _ in range(totalLen)]) - marginLine = f"*{''.join([' ' for _ in range(totalLen - 2)])}*" - - return '\n'.join([topBottomBorder, marginLine, messageLine, marginLine, topBottomBorder]) - -def getSectionMessage(text: str) -> str: - """ - ### This function prints a section message in a specific format. - - #### Params: - - text (str): Title of the section. - - #### Returns: - - (str): Formatted section message. - """ - # Check if provided text's length has exceeded specified limit - textLen = len(text) - remainingLen = SECTION_MESSAGE_LEN - textLen - if remainingLen < 4: - return text - - # Calculating characters around given text - nDashes = remainingLen - 2 - nFinalDashes = int(nDashes / 2) - nInitialDashes = nDashes - nFinalDashes - finalDashes = ''.join(['-' for _ in range(nFinalDashes)]) - initialDashes = ''.join(['-' for _ in range(nInitialDashes)]) - return f"{initialDashes} {text} {finalDashes}" - -def getVersionMessage(short: bool=False) -> str: - """ - ### This function returns the message for the version mode. - - #### Params: - - short (bool): Optional. If True, only Xmipp's version with name will be returned. - - #### Returns: - - (str): Message for version mode. - """ - if short: - return XMIPP_VERSIONS[XMIPP][VERNAME_KEY] - - # Main info - rightSectionStart = 25 - versionType = 'release' if isProductionMode() else getCurrentBranch() - versionMessage = bold(f"Xmipp {XMIPP_VERSIONS[XMIPP][VERSION_KEY]} ({versionType})\n\n") - versionMessage += f"{__getVersionLineWithSpaces('Release date: ', rightSectionStart)}{RELEASE_DATE}\n" - versionMessage += f"{__getVersionLineWithSpaces('Compilation date: ', rightSectionStart)}{getConfigDate(CONFIG_FILE)}\n" - versionMessage += f"{__getVersionLineWithSpaces('System version: ', rightSectionStart)}{getOSReleaseName()}\n" - - # Get sources branch - for source in XMIPP_SOURCES: - sourcePath = os.path.join(SOURCES_PATH, source) - currentCommit = __getCurrentCommit(dir=sourcePath) - branchName = getCurrentBranch(dir=sourcePath) - branchName = branchName if branchName != 'HEAD' else __getCommitName(currentCommit, dir=sourcePath) - sourceText = f"{source} branch: " - versionMessage += f"{__getVersionLineWithSpaces(sourceText, rightSectionStart)}{branchName} ({currentCommit})\n" - versionMessage += "\n" - - # Getting versions from version file - if os.path.exists(VERSION_FILE): - versionDict = parseCmakeVersions(VERSION_FILE) - for package in versionDict: - versionMessage += f"{__getVersionLineWithSpaces(f'{package}: ', rightSectionStart)}{versionDict[package]}\n" - versionMessage = versionMessage[:-1] - else: - warningStr = "This project has not yet been configured, so some detectable dependencies have not been properly detected.\n" - warningStr += f"Run '{MODE_GET_SOURCES}' and then '{MODE_CONFIG_BUILD}' to be able to show all detectable." - versionMessage += yellow(warningStr) - return versionMessage - -####################### AUX FUNCTIONS ####################### -def __branchExists(repo: str, branch: str) -> bool: - """ - ### This function checks if the given branch exists. - - #### Params: - - repo (str): Repository to check the branch from. - - branch (str): Branch to check. - - #### Returns: - - (bool): True if the branch exists, False otherwise. - """ - retCode, _ = runJob(f"git ls-remote --heads {repo}.git {branch} | grep -q refs/heads/{branch}") - return not retCode - -def __logDoneMessage(): - """ - ### This function logs a message shown after completing a task. - """ - logger(green("Done"), forceConsoleOutput=True, substitute=True) - -def __logWorkingMessage(): - """ - ### This function logs a message shown as placeholder for small tasks in progress. - """ - logger(yellow("Working..."), forceConsoleOutput=True) - -def __cloneSourceRepo(repo: str, branch: str=None, path: str='') -> Tuple[int, str]: - """ - ### Clones the given source as a repository in the given branch if exists. Defaults to default branch. - ### If the repository already exists, checks out to specified branch (if provided). - - #### Params: - - repo (str): Source to clone. - - branch (branch): Optional. Branch to clone repo from. - - path (str): Optional. Path to clone the repository into. - - #### Returns: - - (int): 0 if everything worked, or else the return code of the command that failed. - - (str): Output data from the command if it worked or error if it failed. - """ - retCode = 0 - output = '' - __logWorkingMessage() - cloneBranch = __getCloneBranch(repo, branch) - - # If specified branch does not exist, show warning - if branch and not cloneBranch: - warningStr = f"Warning: branch \'{branch}\' does not exist for repository with url {repo}.\n" - warningStr += "Falling back to repository's default branch." - logger(yellow(warningStr), forceConsoleOutput=True, substitute=True) - branch = None - __logWorkingMessage() - - # Move to defined path to clone - currentPath = os.getcwd() - os.chdir(path) - - # Check if repo already exists. If so, checkout instead of clone. - clonedFolder = repo.split("/")[-1] - if os.path.isdir(clonedFolder): - if branch: - os.chdir(clonedFolder) - retCode, output = runJob(f"git checkout {branch}", logOutput=True) - else: - branchStr = f" --branch {branch}" if branch else '' - retCode, output = runJob(f"git clone{branchStr} {repo}.git", logOutput=True) - - # Go back to previous path - os.chdir(currentPath) - - return retCode, output - -def __getCloneBranch(repo: str, branch: str) -> Optional[str]: - """ - ### Returns the branch to clone from in the given repository. - - #### Params: - - repo (str): Repository to clone. - - branch (branch): Branch to clone repo from. - - #### Returns: - - (str | None): The given branch if it is a valid one, or None to indicate default branch. - """ - # If branch exists, use it - if branch and __branchExists(repo, branch): - return branch - - # If repository is xmipp source and current branch is a release, clone from corresponding release - repoName = repo.split("/")[-1] - if repoName in XMIPP_SOURCES: - branchName = getCurrentBranch() - if not branchName or branchName == MASTER_BRANCHNAME or branchName == TAG_BRANCH_NAME: - return XMIPP_VERSIONS[repoName][VERSION_KEY] - - return None - -def __getPredefinedError(realRetCode: int=0, desiredRetCode: int=0) -> int: - """ - ### This function returns the corresponding predefined error for a caller piece of code. - - #### Params: - - realRetCode (int): Optional. Real error code obtained from the process. - - desiredRetCode (int): Optional. Predefined code corresponding to caller code. - """ - return realRetCode if realRetCode == INTERRUPTED_ERROR else desiredRetCode - -def __getCurrentCommit(dir: str="./") -> str: - """ - ### This function returns the current commit short hash of a given repository: - - #### Params: - - dir (str): Optional. Directory to repository. - - #### Returns: - - (str): Current commit short hash, or empty string if it is not a repo or there were errors. - """ - retCode, output = runJob("git rev-parse --short HEAD", cwd=dir) - if retCode or not output: - return '' - return output - -def __getCommitName(commit: str, dir: str="./") -> str: - """ - ### This function returns the name of the commit branch. It can be a branch name or a release name. - - #### Params: - - commit (str): Commit hash - - dir (str): Optional. Directory to repository. - - #### Returns: - - (str): Name of the commit branch or release. - """ - retCode, output = runJob(f"git name-rev {commit}", cwd=dir) - if retCode or not output: - return '' - - # Extract name from commit - return output.replace(commit, "").replace(" ", "") - -def __getVersionLineWithSpaces(text: str, desiredLen: int) -> str: - """ - ### This function returns the given text with additional spaces so that it has the desired length. - - #### Params: - - text (str): Text where extra spaces could be added to. - - desiredLen (int): Target length for the text. - - #### Returns: - - (str): Given text with as much spaces to the right so that it fits the desired length. - """ - # Check if text is already as long or longer than the desired length - textLen = len(text) - if textLen >= desiredLen: - return text - - # Crafting spaces and adding them to the right - spaces = ''.join([' ' for _ in range(desiredLen - textLen)]) - return f"{text}{spaces}" diff --git a/installer/modelsDLTK.py b/installer/modelsDLTK.py deleted file mode 100644 index 23c7871264..0000000000 --- a/installer/modelsDLTK.py +++ /dev/null @@ -1,76 +0,0 @@ -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -import os, sys -from .utils import runJob -from .constants import SCIPION_SOFTWARE_EM, MODELS_URL, DEFAULT_MODELS_DIR -from .logger import blue, red, yellow, logger - - -def addModels(login: str, modelPath: str, update: bool): - modelPath = modelPath.rstrip("/") - modelName = os.path.basename(modelPath) - modelsDir = os.path.dirname(modelPath) - update = '--update' if update else '' - tgzFileModel = f"xmipp_model_{modelName}.tgz" - localFileModel = os.path.join(modelsDir, tgzFileModel) - logger(f"Creating the {tgzFileModel} model.",forceConsoleOutput=True) - runJob(f"tar czf {tgzFileModel} {modelName}", cwd=modelsDir) - - logger(yellow("Warning: Uploading, please BE CAREFUL! This can be dangerous."),forceConsoleOutput=True) - logger(f'You are going to be connected to {login} to write in folder {SCIPION_SOFTWARE_EM}.',forceConsoleOutput=True) - if input("Continue? YES/no\n") != 'YES': - return - - logger(f"Trying to upload the model using {login} as login",forceConsoleOutput=True) - args = f"{login}, {os.path.abspath(localFileModel)}, {SCIPION_SOFTWARE_EM}, {update}" - retCode, log = runJob(f"dist/bin/xmipp_sync_data upload {args}", showCommand=False, showError=True) - if retCode == 0: - logger(f"{modelName} model successfully uploaded! Removing the local .tgz") - runJob(f"rm {localFileModel}") - - - -def downloadDeepLearningModels(dest:str, distPath: str): - if not os.path.exists('dist/bin/xmipp_sync_data'): - logger(red('Xmipp has not been installed. Please, first install it '),forceConsoleOutput=True) - return - if dest == distPath: - modelsPath = os.path.join(dest, 'models') - else: - modelsPath = dest - dataSet = "DLmodels" - - # downloading/updating the DLmodels - if os.path.isdir(modelsPath): - logger("Updating the Deep Learning models (in backgound)") - task = "update" - taskMessage = "updated" - else: - logger("Downloading Deep Learning models (in backgound)") - task = "download" - taskMessage = "downloaded" - retCode, log = runJob(f"dist/bin/xmipp_sync_data {task} {modelsPath} {MODELS_URL} {dataSet}", - showCommand=True, showError=True, showOutput=True) - if retCode == 0: - logger(f"Models successfully {taskMessage}") - diff --git a/installer/parser.py b/installer/parser.py deleted file mode 100644 index d487eb67e1..0000000000 --- a/installer/parser.py +++ /dev/null @@ -1,414 +0,0 @@ -# *************************************************************************** -# * Authors: Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -This module contains a class that extends the capabilities of standard argparser. -""" - -# General imports -import argparse, shutil -from typing import List, Tuple - -# Installer imports -from .constants import (MODES, MODE_ARGS, TAB_SIZE, MODE_EXAMPLES, - MODE_ALL, PARAMS, SHORT_VERSION, LONG_VERSION, DESCRIPTION) -from .logger import yellow, red - -# File specific constants -SECTION_N_DASH = 45 -SECTION_SPACE_MODE_HELP = 2 -SECTION_HELP_START = TAB_SIZE + SECTION_N_DASH + SECTION_SPACE_MODE_HELP -LINE_SIZE_LOWER_LIMIT = int(SECTION_HELP_START * 1.5) - -####################### AUX FUNCTIONS ####################### -def __getLineSize() -> int: - """ - ### This function returns the maximum size for a line. - - ### Returns: - - (int): Maximum line size. - """ - # Getting column size in characters - size = shutil.get_terminal_size().columns - - # Return size with lower limit - return LINE_SIZE_LOWER_LIMIT if size < LINE_SIZE_LOWER_LIMIT else size - -def __fitWordsInLine(words: List[str], sizeLimit: int) -> Tuple[str, List[str]]: - """ - ### This function returns a tuple containig a line with the words from the given list that could fit given the size limit, and the list with the remaining words. - - ### Params: - - words (List[str]): List of words to try to fit into a line. - - sizeLimit (int): Size limit for the text. - - ### Returns: - - (str): Line with the words that were able to fit in it. - - (List[str]): List containing the words that could not fit in the line. - """ - # Initializing line and creating copy of word list - # The copy is made because original list cannot be edited mid iteration - line = '' - remainingWords = words - - # Check if each word fits in the line - for word in words: - # If the line is not empty, len includes extra space - if line: - if len(line + ' ' + word) > sizeLimit: - return line, remainingWords - else: - line += ' ' + word - remainingWords = remainingWords[1:] - else: - # If the first word already exceeds the size limit, - # it means it is a huge word, but we need to print it - # anyways and move on to the next line - if len(word) >= sizeLimit: - return word, remainingWords[1:] - else: - # If word fits, add to line, and remove it from word list - line = word - remainingWords = remainingWords[1:] - - # If we exited the loop, it means all words were introduced in the line - return line, [] - -def __multiLineHelpText(text: str, sizeLimit: int, leftFill: str) -> str: - """ - ### This function returns the given text, formatted in several lines so that it does not exceed the given character limit. - - ### Params: - - text (str): The text to be formatted. - - sizeLimit (int): Size limit for the text. - - leftFill (str): String to add at the left of each new line. - - ### Returns: - - (str): Formatted text. - """ - if len(text) <= sizeLimit: - # If its size is within the limits, return as is - formattedText = text - else: - # If size exceeds limits, split into lines - # We need to calculate each word size to not split the string in the - # middle of a word - textWords = text.split(' ') - - # Initializing list to store lines - lines = [] - - # While there are still words outside of a line, parse them into one. - while textWords: - # Getting new line and removing fitted words in such line - line, textWords = __fitWordsInLine(textWords, sizeLimit) - - # Add line to list - if line: - # If it's not the first line, add the left fill - line = leftFill + line if lines else line - lines.append(line) - - # Join lines into a single string - formattedText = '\n'.join(lines) - - # Return resulting text - return formattedText - -def getFormattingTabs(text: str) -> str: - """ - ### This method returns the given text, formatted to expand tabs into a fixed tab size. - - ### Params: - - text (str): The text to be formatted. - - ### Returns: - - (str): Formatted text. - """ - return text.expandtabs(TAB_SIZE) - -def helpSeparator() -> str: - """ - ### This function returns the line that separates sections inside the help message. - - ### Returns: - - (str): Line that separates sections inside the help message. - """ - dashes = ['-' for _ in range(SECTION_N_DASH)] - return getFormattingTabs(f"\t{''.join(dashes)}\n") - -def textWithLimits(previousText: str, text: str) -> str: - """ - ### This function returns the given text, formatted so that it does not exceed the character limit by line for the param help section. - - ### Params: - - previousText (str): Text inserted before the one to be returned. - - text (str): The text to be formatted. - - ### Returns: - - (str): Formatted text. - """ - # Obtain previous text length - previousLength = len(getFormattingTabs(previousText)) - - # Check if such length exceeds the space reserved for modes and params - if previousLength >= SECTION_HELP_START: - # If so, it means that section space for modes and params - # is too low and should be set to a higher number, but for now we need to print anyways, - # so we reduce space from the one reserved for mode help - remainingSpace = __getLineSize() - previousLength - - # Add minimum fill in space possible - fillInSpace = ' ' - else: - # If such section is within the expected size range, calculate remaining size - # based on the expected help section beginning - remainingSpace = __getLineSize() - SECTION_HELP_START - - # Add fill in space - fillInSpace = ''.join([' ' for _ in range(SECTION_HELP_START - previousLength)]) - - # Format string so it does not exceed size limit - formattedHelp = __multiLineHelpText(text, remainingSpace, ''.join([' ' for _ in range(SECTION_HELP_START)])) - - return previousText + fillInSpace + formattedHelp + '\n' - -def getParamFirstName(paramKey: str) -> str: - """ - ### This function returns the first name of the given param key. Short name has priority over long name. - - ### Params: - - paramKey (str): Key to identify the param. - - ### Returns: - - (str): Formatted text. - """ - return PARAMS[paramKey].get(SHORT_VERSION, PARAMS[paramKey].get(LONG_VERSION, '')) - -####################### HELP FUNCTIONS ####################### -def getModeHelp(mode: str, general: bool=True) -> str: - """ - ### This method returns the help message of a given mode. - - ### Params: - - mode (str): Mode to get help text for. - - general (bool). Optional. If True, only the general help message is displayed. - - ### Returns: - - (str): Help of the mode (empty if mode not found). - """ - # Find mode group containing current mode - for group in list(MODES.keys()): - if mode in list(MODES[group].keys()): - messageList = MODES[group][mode] - if general: - return messageList[0] - else: - return '\n'.join(messageList) - - # If it was not found, return empty string - return '' - -def getParamNames(paramKey: str) -> List[str]: - """ - ### This method returns the list of possible names a given param has. - - ### Params: - - paramKey (str): Key to find the param. - - ### Returns: - - (List[str]): List of all the names of the given param. - """ - nameList = [PARAMS[paramKey].get(SHORT_VERSION, ''), PARAMS[paramKey].get(LONG_VERSION, '')] - return [name for name in nameList if name] - -####################### PARSER CLASS ####################### -class ErrorHandlerArgumentParser(argparse.ArgumentParser): - """ - This class overrides the error function of the standard argument parser - to display better error messages. - """ - ####################### OVERRIDED PUBLIC FUNCTIONS ####################### - def error(self, message): - """ - ### This method prints through stderr the error message and exits with specific return code. - - #### Params: - - message (str): Error message. - """ - # Getting mode and usage help from text - textList = self.prog.split(' ') - mode = textList[-1] - - # If text list only contains one item, mode is generic and - # we need to get the help text - if len(textList) > 1: - textList = ' '.join(textList[:-1]) - extraLineBreak = '\n' - else: - textList = self.format_help() - extraLineBreak = '' - - # Exiting with message - errorMessage = red(f"{mode}: error: {message}\n") - self.exit(1, getFormattingTabs(f"{textList}{extraLineBreak}{errorMessage}")) - -class GeneralHelpFormatter(argparse.HelpFormatter): - """ - This class overrides the default help formatter to display a custom help message. - """ - def __getModeArgsStr(self, mode: str) -> str: - """ - ### This method returns the args text for a given mode. - - ### Params: - - mode (str): Mode to get args text for. - - ### Returns: - - (str): Args text for given mode. - """ - # Getting argument list for the mode - argList = MODE_ARGS[mode] - - # Formatting every element - paramNames = [] - for param in argList: - paramName = getParamFirstName(param) - if paramName: - paramNames.append(f'[{paramName}]') - - # Returning all formatted param names as a string - return ' '.join(paramNames) - - def __getModeArgsAndHelpStr(self, previousText: str, mode: str) -> str: - """ - ### This method returns the args and help text for a given mode. - - ### Params: - - previousText (str): Text inserted before the one to be returned. - - mode (str): Mode to get help text for. - - ### Returns: - - (str): Args and help text for given mode. - """ - # Initializing help string to format - modeHelpStr = '' - - # Find mode group containing current mode - modeHelpStr = getModeHelp(mode) - - # Return formatted text formed by the previous text, - # the args for the mode, and its help text - return textWithLimits(previousText + self.__getModeArgsStr(mode), modeHelpStr) - - def format_help(self): - """ - ### This method prints the help message of the argument parser. - """ - # Base message - helpMessage = "Run Xmipp's installer script\n\nUsage: xmipp [options]\n" - - # Add every section - for section in list(MODES.keys()): - # Adding section separator and section name - helpMessage += helpSeparator() + f"\t# {section} #\n\n" - - # Adding help text for every mode in each section - for mode in list(MODES[section].keys()): - helpMessage += self.__getModeArgsAndHelpStr(f"\t{mode} ", mode) - - # Adding epilog and returning to print - epilog = "Example 1: ./xmipp\n" - epilog += "Example 2: ./xmipp compileAndInstall -j 4\n" - helpMessage += '\n' + epilog - - # Adding note about mode specific help - noteMessage = "Note: You can also view a specific help message for each mode with \"./xmipp [mode] -h\".\n" - noteMessage += f"Example: ./xmipp {MODE_ALL} -h\n" - helpMessage += yellow(noteMessage) - return getFormattingTabs(helpMessage) - -class ModeHelpFormatter(argparse.HelpFormatter): - """ - This class overrides the default help formatter to display a custom help message deppending on the mode selected. - """ - def __argsContainOptional(self, argNames: List[str]) -> bool: - """ - ### This method returns True if the param name list contains at least one optional param. - - ### Params: - - argNames (List[str]): List containing the param names. - - ### Returns: - - (bool): True if there is at least one optional param. False otherwise. - """ - # For every param name, check if starts with '-' - for name in argNames: - if name.startswith('-'): - return True - - # If execution gets here, there were no optional params - return False - - def format_help(self): - """ - ### This method prints the help message of the argument parser. - """ - # Getting the selected mode from the parent help message - # Message received is the format_help of the main parser's - # formatter, adding the mode at the end - mode = self._prog.split(' ')[-1] - - # Initialize the help message - helpMessage = getModeHelp(mode, general=False) + '\n\n' - - # Get mode args - args = MODE_ARGS[mode] - - # Add extra messages deppending on if there are args - optionsStr = '' - separator = '' - if len(args) > 0: - argNames = [getParamFirstName(argName) for argName in args] - if self.__argsContainOptional(argNames): - helpMessage += yellow("Note: only params starting with '-' are optional. The rest are required.\n") - optionsStr = ' [options]' - separator = helpSeparator() + '\t# Options #\n\n' - helpMessage += f'Usage: xmipp {mode}{optionsStr}\n{separator}' - - # Adding arg info - for arg in args: - helpMessage += textWithLimits('\t' + ', '.join(getParamNames(arg)), PARAMS[arg][DESCRIPTION]) - - # Adding a few examples - examples = MODE_EXAMPLES[mode] - for i in range(len(examples)): - numberStr = '' if len(examples) == 1 else f' {i+1}' - helpMessage += f"\nExample{numberStr}: {examples[i]}" - - # If any test were added, add extra line break - if len(examples) > 0: - helpMessage += '\n' - - return getFormattingTabs(helpMessage) - \ No newline at end of file diff --git a/installer/test.py b/installer/test.py deleted file mode 100644 index b1e516621b..0000000000 --- a/installer/test.py +++ /dev/null @@ -1,85 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto Garcia (alberto.garcia@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ -""" -This module contains the necessary functions to run and manage the test. -""" - -# General imports -from os import environ, path - -# Self imports -from .constants import XMIPP, SCIPION_TESTS_URLS, CONFIG_FILE, XMIPP_USE_CUDA, XMIPP_LINK_TO_SCIPION, ENVIROMENT_ERROR -from .logger import blue, red, logger -from .utils import runJob, runStreamingJob -from .config import readConfig -from .main import handleRetCode - -####################### COMMAND FUNCTIONS ####################### - - -def runTests(testNames): - """ - ### This function fetches the sources needed for Xmipp to compile. - - #### Params: - - branch (str): Optional. Branch to clone the sources from. - """ - xmippSrc = environ.get('XMIPP_SRC', None) - if xmippSrc and path.isdir(xmippSrc): - environ['PYTHONPATH'] = ':'.join([ - path.join(environ['XMIPP_SRC'], XMIPP), - environ.get('PYTHONPATH', '')]) - testsPath = path.join(environ['XMIPP_SRC'], XMIPP, 'tests') - dataSetPath = path.join(testsPath, 'data') - - else: - handleRetCode(ENVIROMENT_ERROR, predefinedErrorCode=ENVIROMENT_ERROR, sendAPI=False) - - # downloading/updating the dataset - dataset = 'xmipp_programs' - if path.isdir(dataSetPath): - logger(blue("Updating the test files")) - task = "update" - showO = False - else: - logger(blue("Downloading the test files")) - task = "download" - showO = True - args = "%s %s %s" % ("tests/data", SCIPION_TESTS_URLS, dataset) - runJob("xmipp_sync_data %s %s" % (task, args), cwd='src/xmipp', showOutput=showO) - - configDict = readConfig(CONFIG_FILE) if path.exists(CONFIG_FILE) else {} - CUDA = configDict.get(XMIPP_USE_CUDA) - noCudaStr = '--noCuda' if CUDA == 'OFF' else '' - - logger(" Tests to do: %s" % ', '.join(testNames)) - - # if configDict.get(XMIPP_LINK_TO_SCIPION) == 'ON': - # pythonExe = 'scipion3 python' - # else: - # pythonExe = 'python3' - - pythonExe = 'python3' #TODO should be scipion3? - testScripts = path.dirname(environ['XMIPP_TEST_DATA']) - runStreamingJob(cmd="%s test.py %s %s" % (pythonExe, testNames, noCudaStr), cwd=testScripts, showOutput=True, showError=True) - diff --git a/installer/utils.py b/installer/utils.py deleted file mode 100644 index 8797a6cca2..0000000000 --- a/installer/utils.py +++ /dev/null @@ -1,340 +0,0 @@ -# *************************************************************************** -# * Authors: Alberto García (alberto.garcia@cnb.csic.es) -# * Martín Salinas (martin.salinas@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - -""" -Module containing useful functions used by the installation process. -""" - -# General imports -import os, multiprocessing -from typing import List, Tuple, Callable, Any, Optional -from subprocess import Popen, PIPE -from threading import Thread -from io import BufferedReader - -# Installer imports -from .constants import XMIPP, VERNAME_KEY, XMIPP_VERSIONS, INTERRUPTED_ERROR -from .constants.versions import LATEST_RELEASE_NAME -from .logger import blue, red, logger - -####################### RUN FUNCTIONS ####################### -def runJob(cmd: str, cwd: str='./', showOutput: bool=False, showError: bool=False, - showCommand: bool=False, substitute: bool=False, logOutput: bool=False) -> Tuple[int, str]: - """ - ### This function runs the given command. - - #### Params: - - cmd (str): Command to run. - - cwd (str): Optional. Path to run the command from. Default is current directory. - - showOutput (bool): Optional. If True, output is printed. - - showError (bool): Optional. If True, errors are printed. - - showCommand (bool): Optional. If True, command is printed in blue. - - substitute (bool): Optional. If True, output will replace previous line. - - logOutput (bool): Optional. If True, output will be stored in the log. - - #### Returns: - - (int): Return code. - - (str): Output of the command, regardless of if it is an error or regular output. - """ - # Printing command if specified - __logToSelection(blue(cmd), sendToLog=logOutput, sendToTerminal=showCommand, substitute=substitute) - - # Running command - process = Popen(cmd, cwd=cwd, env=os.environ, stdout=PIPE, stderr=PIPE, shell=True) - try: - process.wait() - except KeyboardInterrupt: - return INTERRUPTED_ERROR, "" - - # Defining output string - retCode = process.returncode - output, err = process.communicate() - outputStr = output.decode() if not retCode and output else err.decode() - outputStr = outputStr[:-1] if outputStr.endswith('\n') else outputStr - - # Printing output if specified - if not retCode: - __logToSelection(f"{outputStr}", sendToLog=logOutput, sendToTerminal=showOutput, substitute=substitute) - - # Printing errors if specified - if retCode and showError: - if logOutput: - logger.logError(outputStr) - else: - print(red(outputStr)) - - # Returing return code - return retCode, outputStr - -def runInsistentJob(cmd: str, cwd: str='./', showOutput: bool=False, showError: bool=False, showCommand: bool=False, nRetries: int=5) -> Tuple[int, str]: - """ - ### This function runs the given network command and retries it the number given of times until one of the succeeds or it fails for all the retries. - - #### Params: - - cmd (str): Command to run. - - cwd (str): Optional. Path to run the command from. Default is current directory. - - showOutput (bool): Optional. If True, output is printed. - - showError (bool): Optional. If True, errors are printed. - - showCommand (bool): Optional. If True, command is printed in blue. - - nRetries (int): Optional. Maximum number of retries for the command. - - #### Returns: - - (int): Return code. - - (str): Output of the command, regardless of if it is an error or regular output. - """ - # Running command up to nRetries times (improves resistance to small network errors) - for _ in range(nRetries): - retCode, output = runJob(cmd, cwd=cwd) - # Break loop if success was achieved - if retCode == 0: - break - - # Enforce message showing deppending on value - if showCommand: - print(blue(cmd)) - if showOutput: - print('{}\n'.format(output)) - if showError: - print(red(output)) - - # Returning output and return code - return retCode, output - -def runParallelJobs(funcs: List[Tuple[Callable, Tuple[Any]]], nJobs: int=multiprocessing.cpu_count()) -> List: - """ - ### This function runs the given command list in parallel. - - #### Params: - - funcs (list(tuple(callable, tuple(any)))): Functions to run with parameters, if there are any. - - #### Returns: - - (list): List containing the return of each function. - """ - # Creating a pool of n concurrent jobs - with multiprocessing.Pool(nJobs) as p: - # Run each function and obtain results - results = p.starmap(__runLambda, funcs) - - # Return obtained result list - return results - -def runStreamingJob(cmd: str, cwd: str='./', showOutput: bool=False, showError: bool=False, substitute: bool=False) -> int: - """ - ### This function runs the given command and shows its output as it is being generated. - - #### Params: - - cmd (str): Command to run. - - cwd (str): Optional. Path to run the command from. Default is current directory. - - showOutput (bool): Optional. If True, output is printed. - - showError (bool): Optional. If True, errors are printed. - - substitute (bool): Optional. If True, output will replace previous line. - - #### Returns: - - (int): Return code. - """ - # Create a Popen instance and error stack - logger(cmd) - process = Popen(cmd, cwd=cwd, stdout=PIPE, stderr=PIPE, shell=True) - - # Create and start threads for handling stdout and stderr - threadOut = Thread(target=__handleOutput, args=(process.stdout, showOutput, substitute)) - threadErr = Thread(target=__handleOutput, args=(process.stderr, showError, substitute, True)) - threadOut.start() - threadErr.start() - - # Wait for execution, handling keyboard interruptions - try: - process.wait() - threadOut.join() - threadErr.join() - except (KeyboardInterrupt): - process.returncode = INTERRUPTED_ERROR - - return process.returncode - -####################### GIT FUNCTIONS ####################### -def getCurrentBranch(dir: str='./') -> str: - """ - ### This function returns the current branch of the repository of the given directory or empty string if it is not a repository or a recognizable tag. - - #### Params: - - dir (str): Optional. Directory of the repository to get current branch from. Default is current directory. - - #### Returns: - - (str): The name of the branch, 'HEAD' if a tag, or empty string if given directory is not a repository or a recognizable tag. - """ - # Getting current branch name - retcode, branchName = runJob("git rev-parse --abbrev-ref HEAD", cwd=dir) - - # If there was an error, we are in no branch - return branchName if not retcode else '' - -def isProductionMode(dir: str='./') -> bool: - """ - ### This function returns True if the current Xmipp repository is in production mode. - - #### Params: - - dir (str): Optional. Directory of the repository where the check will happen. Default is current directory. - - #### Returns: - - (bool): True if the repository is in production mode. False otherwise. - """ - currentBranch = getCurrentBranch(dir=dir) - return currentBranch is None or currentBranch == XMIPP_VERSIONS[XMIPP][VERNAME_KEY] - -def isTag(dir: str='./') -> bool: - """ - ### This function returns True if the current Xmipp repository is in a tag. - - #### Params: - - dir (str): Optional. Directory of the repository where the check will happen. Default is current directory. - - #### Returns: - - (bool): True if the repository is a tag. False otherwise. - """ - currentBranch = getCurrentBranch(dir=dir) - return not currentBranch or currentBranch == "HEAD" - - -def getCurrentName(): - """ - ### This function returns the current branch of the repository of the given directory or the name of the tag. - - #### Params: - - dir (str): Optional. Directory of the repository where the check will happen. Default is current directory. - - #### Returns: - - (str): The name of the branch or the tag. - """ - if isTag(): - return LATEST_RELEASE_NAME - else: - return getCurrentBranch() - - - -def isBranchUpToDate(dir: str='./') -> bool: - """ - ### This function returns True if the current branch is up to date, or False otherwise or if some error happened. - - #### Params: - - dir (str): Optional. Directory of the repository to get current branch from. Default is current directory. - - #### Returns: - - (bool): True if the current branch is up to date, or False otherwise or if some error happened. - """ - # Getting current branch - currentBranch = getCurrentBranch(dir=dir) - - # Check if previous command succeeded - if currentBranch is None: - return False - - # Update branch - retCode = runInsistentJob("git fetch")[0] - - # Check if command succeeded - if retCode != 0: - return False - - # Get latest local commit - localCommit = runJob(f"git rev-parse {currentBranch}")[1] - - # Get latest remote commit - retCode, remoteCommit = runInsistentJob(f"git rev-parse origin/{currentBranch}") - - # Check if command succeeded - if retCode != 0: - return False - - # Return commit comparison - return localCommit == remoteCommit - -####################### VERSION FUNCTIONS ####################### -def getPackageVersionCmd(packageName: str) -> Optional[str]: - """ - ### Retrieves the version of a package or program by executing '[packageName] --version' command. - - Params: - - packageName (str): Name of the package or program. - - Returns: - - (str | None): Version information of the package or None if not found or errors happened. - """ - # Running command - retCode, output = runJob(f'{packageName} --version') - - # Check result if there were no errors - return output if retCode == 0 else None - -####################### AUX FUNCTIONS (INTERNAL USE ONLY) ####################### -def __handleOutput(stream: BufferedReader, show: bool=False, substitute: bool=False, err: bool=False): - """ - ### This function receives a process output stream and logs its lines. - - #### Params: - - stream (BufferedReader): Function to run. - - show (bool): Optional. If True, output will also be printed through terminal. - - substitute (bool): Optional. If True, output will replace previous line. Only used when show is True. - - err (bool): Optional. If True, the stream contains an error. Otherwise, it is regular output. - """ - # If print through terminal is enabled with substitution, add a first line break - if show and substitute: - print("") - - # Print all lines in the process output - for line in iter(stream.readline, b''): - line = line.decode().replace("\n", "") - if err: - line = red(line) - logger(line, forceConsoleOutput=show, substitute=substitute) - -def __runLambda(function: Callable, args: Tuple[Any]=()): - """ - ### This function is used to run other functions (intented for use inside a worker pool, so it can be picked). - - #### Params: - - function (callable): Function to run. - - args (tuple(any)): Optional. Function arguments. - - #### Returns: - - (Any): Return value/(s) of the called function. - """ - return function(*args) - -def __logToSelection(message: str, sendToLog: bool=True, sendToTerminal: bool=False, substitute: bool=False): - """ - ### This function logs the given message into the selected logging platform. - - #### Params: - - message (str): Message to log. - - sendToLog (bool): Optional. If True, message is sent to the logger (into file). - - sendToTerminal (bool): Optional. If True, message is sent to terminal. - - substitute (bool): Optional. If True, message will replace last terminal printed message. Only used when all other variables are True. - """ - if sendToLog: - logger(message, forceConsoleOutput=sendToTerminal, substitute=substitute) - else: - if sendToTerminal: - print(message) diff --git a/version-info.json b/version-info.json new file mode 100644 index 0000000000..bc221ca791 --- /dev/null +++ b/version-info.json @@ -0,0 +1,12 @@ +{ + "xmipp": { + "version_number": "4.0.0", + "version_name": "v4.0.0-Apoferritin", + "release_date": "17/11/2025" + }, + "sources_target_tag": { + "xmippCore": "v4", + "xmippViz": "v4" + } +} + diff --git a/xmipp b/xmipp index 84c75adaba..5879e325e3 100755 --- a/xmipp +++ b/xmipp @@ -23,30 +23,11 @@ # * e-mail address 'scipion@cnb.csic.es' # ***************************************************************************/ -# General imports -import argparse, sys, os, multiprocessing -from typing import Dict +import os +import shutil +import subprocess +import sys -# Installer imports -from installer.constants import (MODE_ALL, MODE_COMPILE_AND_INSTALL, MODE_CONFIG, MODE_CLEAN_BIN, - MODE_CLEAN_ALL, MODE_VERSION, MODE_GET_MODELS, MODE_TEST, MODE_GIT, MODE_ADD_MODEL, MODE_CONFIG_BUILD, - COMMON_USAGE_HELP_MESSAGE, DEFAULT_MODELS_DIR, CONFIG_FILE, CMAKE_INSTALL_PREFIX, - CMAKE_CONFIGURE_ERROR, CMAKE_COMPILE_ERROR, CMAKE_INSTALL_ERROR, PARAM_LOGIN, PARAM_SHORT, - PARAM_JOBS, PARAM_BRANCH, PARAM_GIT_COMMAND, PARAM_TEST_PRO, PARAM_TEST_PRO, PARAM_TEST_FUNC, PARAM_MODEL_PATH, - PARAM_MODELS_DIRECTORY, PARAM_KEEP_OUTPUT, PARAM_SHOW_TESTS, PARAM_TEST_NAME, PARAM_UPDATE, - PARAM_OVERWRITE, BUILD_PATH, INSTALL_PATH, BUILD_TYPE, SOURCES_PATH, XMIPP_SOURCES, XMIPP, PARAM_PRODUCTION, - LOG_FILE, CMAKE_ERROR, MODE_GET_SOURCES, VERSION_FILE, PARAMS, LONG_VERSION, SHORT_VERSION, SEND_INSTALLATION_STATISTICS) -from installer.utils import runStreamingJob, runJob -from installer.parser import ModeHelpFormatter, GeneralHelpFormatter, ErrorHandlerArgumentParser, getParamNames -from installer.config import readConfig, writeConfig -from installer.cmake import getCMake, getCMakeVarsStr -from installer.main import getSources, exitXmipp, handleRetCode, getSectionMessage, getSuccessMessage, getVersionMessage -from installer.logger import logger, yellow, red, blue, green -from installer.api import sendApiPOST -from installer.test import runTests -from installer import modelsDLTK - -####################### EXECUTION MODES ####################### def __getProjectRootDir() -> str: """ ### Returns the root directory of Xmipp. @@ -56,395 +37,22 @@ def __getProjectRootDir() -> str: """ return os.path.dirname(os.path.abspath(__file__)) -####################### EXECUTION MODES ####################### -def modeAddModel(args: argparse.Namespace): - """ - ### Checks the params for execution mode "addModel" and then runs it. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - """ - if not os.path.isdir(args.modelPath): - logger(" is not a directory. Please, check the path. \n" - "The name of the model will be the name of that folder.\n") - else: - modelsDLTK.addModels(login=args.login, modelPath=args.modelPath, update=args.update) - -def modeCleanBin(): - """ - ### Removes all compiled binaries. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - """ - confirmationText = 'y' - warningStr = "WARNING: This will DELETE from {} all *.so, *.os and *.o files. Also the *.pyc and *.dblite files".format(SOURCES_PATH) - warningStr += f"\nIf you are sure you want to do this, type '{confirmationText}' (case sensitive):" - logger(yellow(warningStr), forceConsoleOutput=True) - try: - userInput = input() - except KeyboardInterrupt: - userInput = '' - logger("", forceConsoleOutput=True) - if userInput == confirmationText: - runJob(f"find {SOURCES_PATH}/* -name \"*.so\" -exec rm -rf {{}}", showCommand=True) - runJob(f"find {SOURCES_PATH}/* -name \"*.os\" -exec rm -rf {{}}", showCommand=True) - runJob(f"find {SOURCES_PATH}/* -name \"*.o\" -exec rm -rf {{}}", showCommand=True) - runJob("find . -iname \"*.pyc\" -delete", showCommand=True) - runJob("find . -iname \"*.dblite\" -delete", showCommand=True) - runJob(f"find {os.path.join(SOURCES_PATH, XMIPP, 'applications', 'programs')} --type d -empty", showCommand=True) - runJob(f"rm -rf {BUILD_PATH}", showCommand=True) - else: - logger(red("Operation cancelled."), forceConsoleOutput=True) - -def modeCleanAll(): - """ - ### Removes all compiled binaries and cloned sources. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - """ - # Print warning text and await input - confirmationText = 'YeS' - warningStr = "WARNING: This will DELETE ALL content from src and build, also de xmipp.conf file.\n" - warningStr += "\tNotice that if you have unpushed changes, they will be deleted.\n" - warningStr += f"\nIf you are sure you want to do this, type '{confirmationText}' (case sensitive):" - logger(yellow(warningStr), forceConsoleOutput=True) - - try: - userInput = input() - except KeyboardInterrupt: - userInput = '' - logger("", forceConsoleOutput=True) - if userInput == confirmationText: - # Get xmipp sources - xmippSources = [os.path.join(SOURCES_PATH, source) for source in XMIPP_SOURCES] - # Get installation path - configDict = readConfig(CONFIG_FILE) if os.path.exists(CONFIG_FILE) else {} - installDir = configDict.get(CMAKE_INSTALL_PREFIX, INSTALL_PATH) - runJob(f"rm -rf {BUILD_PATH} {installDir} {' '.join(xmippSources)} {CONFIG_FILE}", showCommand=True) - else: - logger(red("Operation cancelled."), forceConsoleOutput=True) - -def modeCompileAndInstall(args: argparse.Namespace, configDict: Dict={}): - """ - ### Checks the params for execution mode "compileAndInstall" and then runs it. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - - configDict (dict): Optional. If provided, it will use it's variables. If not, it will read from config file. - """ - # If config variable dictionary is not received this mode is the first being executed - if not configDict: - logger.startLogFile(LOG_FILE) - configDict = readConfig(CONFIG_FILE) - if not getCMake(configDict): - apiPost = True if configDict.get(SEND_INSTALLATION_STATISTICS) == 'ON' else False - handleRetCode(CMAKE_ERROR, predefinedErrorCode=CMAKE_ERROR, sendAPI=apiPost) - - # Compile with CMake - cmakeExecutable = getCMake(configDict) - logger(getSectionMessage("Compiling with CMake"), forceConsoleOutput=True) - retCode = runStreamingJob(f"{cmakeExecutable} --build {BUILD_PATH} --config {BUILD_TYPE} -j {args.jobs}", - showOutput=True, substitute=True) - apiPost = True if configDict.get( - SEND_INSTALLATION_STATISTICS) == 'ON' else False - handleRetCode(retCode, predefinedErrorCode=CMAKE_COMPILE_ERROR, sendAPI=apiPost) - - # Install with CMake - logger(getSectionMessage("Installing with CMake"), forceConsoleOutput=True) - retCode = runStreamingJob(f"{cmakeExecutable} --install {BUILD_PATH} --config {BUILD_TYPE}", - showOutput=True, substitute=True) - handleRetCode(retCode, predefinedErrorCode=CMAKE_INSTALL_ERROR, sendAPI=apiPost) - -def modeConfigBuild(configDict: Dict={}, sendAPIPost: bool=False): - """ - ### Configures the project using CMake. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - - configDict (dict): Optional. If provided, it will use it's variables. If not, it will read from config file. - """ - # If config variable dictionary is not received this mode is the first being executed - if not configDict: - logger.startLogFile(LOG_FILE) - configDict = readConfig(CONFIG_FILE) if os.path.exists(CONFIG_FILE) else modeConfig() - - # Check if CMake exists - cmakeExecutable = getCMake(configDict) - if not getCMake(configDict): - handleRetCode(CMAKE_ERROR, predefinedErrorCode=CMAKE_ERROR, sendAPI=sendAPIPost) - - logger(getSectionMessage("Configuring with CMake"), forceConsoleOutput=True) - configureCmd = f"{cmakeExecutable} -S . -B {BUILD_PATH} -D CMAKE_BUILD_TYPE={BUILD_TYPE}" - configureCmd += f" {getCMakeVarsStr(configDict)}" - retCode = runStreamingJob(configureCmd, showOutput=True, substitute=True) - handleRetCode(retCode, predefinedErrorCode=CMAKE_CONFIGURE_ERROR, sendAPI=sendAPIPost) - -def modeConfig(overwrite: bool=False) -> Dict: - """ - ### Generates a template config file. - - #### Params: - - overwrite (bool): If True, file is created from scratch with default values. - - #### Returns: - - (dict): Dictionary containig all config variables. - """ - configDict = {} - logger(getSectionMessage("Managing config file"), forceConsoleOutput=True) - if not overwrite and os.path.exists(CONFIG_FILE): - logger('Reading config file...', forceConsoleOutput=True) - configDict = readConfig(CONFIG_FILE) - else: - logger('Generating config file from scratch with default values...', forceConsoleOutput=True) - writeConfig(CONFIG_FILE, configDict=configDict) - logger(green("Done"), forceConsoleOutput=True) - return readConfig(CONFIG_FILE) - -def modeGetModels(args: argparse.Namespace): - """ - ### Checks the params for execution mode "getModels" and then runs it. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - """ - distPath = os.path.join(__getProjectRootDir(), DEFAULT_MODELS_DIR) - modelsDLTK.downloadDeepLearningModels(args.directory, distPath) - -def modeGetSources(args: argparse.Namespace): - """ - ### Downloads all Xmipp's sources. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - """ - # Clone sources from specified branch - if hasattr(args, "production"): - getSources(branch=args.branch, production=args.production) - else: - getSources(branch=args.branch) - -def modeGit(args: argparse.Namespace): - """ - ### Executes the given git command into all xmipp source repositories. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - """ - cmd = f"git {' '.join(args.command)}" - logger(f"Running command '{cmd}' for all xmipp sources...", forceConsoleOutput=True) - XMIPP_SOURCES.insert(0, XMIPP) - for source in XMIPP_SOURCES: - logger("", forceConsoleOutput=True) - # Check if source exists to perform command, else skip - sourcePath = os.path.abspath(os.path.join(SOURCES_PATH, source)) - if not os.path.exists(sourcePath): - logger(yellow(f"WARNING: Source {source} does not exist in path {sourcePath}. Skipping."), forceConsoleOutput=True) - continue - logger(blue(f"Running command for {source} in path {sourcePath}..."), forceConsoleOutput=True) - runJob(cmd, cwd=sourcePath, showOutput=True, showError=True) - -def modeTest(args: argparse.Namespace): - """ - ### Checks the params for execution mode "test" and then runs it. - - #### Params: - - parser (ErrorHandlerArgumentParser): Parser object used to parse the arguments. - - args (Namespace): Command line arguments parsed by argparse library. - """ - if args.show == True: - logger("Showing test--------------------------------------", forceConsoleOutput=True) - runTests(PARAMS[PARAM_SHOW_TESTS][LONG_VERSION]) - elif args.allPrograms: - logger("Running all tests--------------------------------------", forceConsoleOutput=True) - runTests(PARAMS[PARAM_TEST_PRO][LONG_VERSION]) - elif args.allFuncs: - runTests(PARAMS[PARAM_TEST_FUNC][LONG_VERSION]) - - elif args.testName: - logger("Running test {}-------------------------------".format(args.testName), forceConsoleOutput=True) - runTests(args.testName) - -def modeVersion(args: argparse.Namespace): - """ - ### Checks the params for execution mode "version" and then runs it. - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - """ - logger(getVersionMessage(short=args.short), forceConsoleOutput=True) - -def modeAll(args: argparse.Namespace): - """ - ### Runs get sources & modes config, configBuild, and compileAndInstall - - #### Params: - - args (Namespace): Command line arguments parsed by argparse library. - """ - # Initiate logger - logger.startLogFile(LOG_FILE) - - # Read and/or write variables from config file - configDict = modeConfig() - - # Clone sources from specified branch - modeGetSources(args) - - # Configure with CMake - modeConfigBuild(configDict=configDict, sendAPIPost=True) - - # Compile and install with CMake - modeCompileAndInstall(args, configDict=configDict) - -def runSelectedMode(parser: ErrorHandlerArgumentParser, args: argparse.Namespace): - """ - ### This function runs the selected execution mode. - - #### Params: - - parser (ErrorHandlerArgumentParser): Parser object used to parse the arguments. - - args (Namespace): Command line arguments parsed by argparse library. - """ - sendAPI = False - if args.mode == MODE_ADD_MODEL: - modeAddModel(args) - elif args.mode == MODE_ALL: - sendAPI = True - modeAll(args) - elif args.mode == MODE_CLEAN_ALL: - modeCleanAll() - elif args.mode == MODE_CLEAN_BIN: - modeCleanBin() - elif args.mode == MODE_COMPILE_AND_INSTALL: - modeCompileAndInstall(args) - sendAPI = True - elif args.mode == MODE_CONFIG_BUILD: - sendAPI = True - modeConfigBuild() - elif args.mode == MODE_CONFIG: - modeConfig(overwrite=args.overwrite) - elif args.mode == MODE_GET_MODELS: - modeGetModels(args) - elif args.mode == MODE_GET_SOURCES: - modeGetSources(args) - elif args.mode == MODE_GIT: - modeGit(args) - elif args.mode == MODE_TEST: - modeTest(args) - elif args.mode == MODE_VERSION: - modeVersion(args) - else: - # If method was none of the above, exit with error - logger(red(f"Mode \"{args.mode}\" not recognized. {COMMON_USAGE_HELP_MESSAGE}"), forceConsoleOutput=True) - exitXmipp(retCode=1) - - # Send API message - try: - apiPost = True if readConfig(CONFIG_FILE).get( - SEND_INSTALLATION_STATISTICS) == 'ON' else False - if sendAPI and apiPost and os.path.exists(VERSION_FILE): - sendApiPOST() - except FileNotFoundError: - pass - - # Print success message for specific modes - if args.mode == MODE_ALL or args.mode == MODE_COMPILE_AND_INSTALL: - logger(getSuccessMessage(), forceConsoleOutput=True) - exitXmipp() - ####################### MAIN EXECUTION THREAD ####################### if __name__ == "__main__": """ Calls main function when executed. """ - # Defining default jobs: 120% current thread count (not all jobs take 100% of CPU time continuously) - JOBS = multiprocessing.cpu_count() + int(multiprocessing.cpu_count() * 0.2) - - # Creating parser to parse the command-line arguments - parser = ErrorHandlerArgumentParser(formatter_class=GeneralHelpFormatter, prog="xmipp") - - # Adding subparsers to have other variables deppending on the value of the mode - subparsers = parser.add_subparsers(dest="mode") - - # Arguments for mode 'addModel' - addModelSubparser = subparsers.add_parser(MODE_ADD_MODEL, formatter_class=ModeHelpFormatter) - addModelSubparser.add_argument(*getParamNames(PARAM_LOGIN)) - addModelSubparser.add_argument(*getParamNames(PARAM_MODEL_PATH)) - addModelSubparser.add_argument(*getParamNames(PARAM_UPDATE), action='store_true') - - # Arguments for mode 'all' - allSubparser = subparsers.add_parser(MODE_ALL, formatter_class=ModeHelpFormatter) - allSubparser.add_argument(*getParamNames(PARAM_JOBS), type=int, default=JOBS) - allSubparser.add_argument(*getParamNames(PARAM_BRANCH)) - allSubparser.add_argument(*getParamNames(PARAM_PRODUCTION)) - allSubparser.add_argument(*getParamNames(PARAM_KEEP_OUTPUT), action='store_true') - - # Arguments for mode 'cleanAll' - cleanAllSubparser = subparsers.add_parser(MODE_CLEAN_ALL, formatter_class=ModeHelpFormatter) - - # Arguments for mode 'cleanBin' - cleanBinSubparser = subparsers.add_parser(MODE_CLEAN_BIN, formatter_class=ModeHelpFormatter) - - # Arguments for mode 'compileAndInstall' - compileAndInstallSubparser = subparsers.add_parser(MODE_COMPILE_AND_INSTALL, formatter_class=ModeHelpFormatter) - compileAndInstallSubparser.add_argument(*getParamNames(PARAM_JOBS), type=int, default=JOBS) - compileAndInstallSubparser.add_argument(*getParamNames(PARAM_BRANCH)) - compileAndInstallSubparser.add_argument(*getParamNames(PARAM_KEEP_OUTPUT), action='store_true') - - # Arguments for mode 'configBuild' - buildConfigSubparser = subparsers.add_parser(MODE_CONFIG_BUILD, formatter_class=ModeHelpFormatter) - buildConfigSubparser.add_argument(*getParamNames(PARAM_KEEP_OUTPUT), action='store_true') - - # Arguments for mode 'config' - configSubparser = subparsers.add_parser(MODE_CONFIG, formatter_class=ModeHelpFormatter) - configSubparser.add_argument(*getParamNames(PARAM_OVERWRITE), action='store_true') - - # Arguments for mode 'getModels' - getModelsSubparser = subparsers.add_parser(MODE_GET_MODELS, formatter_class=ModeHelpFormatter) - getModelsSubparser.add_argument(*getParamNames(PARAM_MODELS_DIRECTORY), default=os.path.join(__getProjectRootDir(), DEFAULT_MODELS_DIR)) - - # Arguments for mode 'getSources' - getSourcesSubparser = subparsers.add_parser(MODE_GET_SOURCES, formatter_class=ModeHelpFormatter) - getSourcesSubparser.add_argument(*getParamNames(PARAM_BRANCH)) - getSourcesSubparser.add_argument(*getParamNames(PARAM_KEEP_OUTPUT), action='store_true') - - # Arguments for mode 'git' - gitSubparser = subparsers.add_parser(MODE_GIT, formatter_class=ModeHelpFormatter) - gitSubparser.add_argument(*getParamNames(PARAM_GIT_COMMAND), nargs='+') - - # Arguments for mode 'test' - testSubparser = subparsers.add_parser(MODE_TEST, formatter_class=ModeHelpFormatter) - testSubparser.add_argument(*getParamNames(PARAM_TEST_NAME), nargs='?', default=None) - testSubparser.add_argument(*getParamNames(PARAM_TEST_PRO), action='store_true') - testSubparser.add_argument(*getParamNames(PARAM_TEST_FUNC), action='store_true') - testSubparser.add_argument(*getParamNames(PARAM_SHOW_TESTS), action='store_true') - - # Arguments for mode 'version' - versionSubparser = subparsers.add_parser(MODE_VERSION, formatter_class=ModeHelpFormatter) - versionSubparser.add_argument(*getParamNames(PARAM_SHORT), action='store_true') - - # Applying default mode value if needed - if len(sys.argv) == 1 or ( - len(sys.argv) > 1 and - sys.argv[1].startswith('-') and - '-h' not in sys.argv and - '--help' not in sys.argv): - sys.argv.insert(1, MODE_ALL) - - # Parse arguments - args = parser.parse_args() - - # Error control for number of jobs - if hasattr(args, 'jobs') and args.jobs < 1: - parser.error(f"Wrong job number \"{args.jobs}\". Number of jobs has to be 1 or greater.") - - # Error control for branch - if hasattr(args, "branch") and args.branch is not None and len(args.branch.split(' ')) > 1: - parser.error(f"Incorrect branch name \"{args.branch}\". Branch names can only be one word long.") - - if hasattr(args, "keep_output") and args.keep_output: - logger.setAllowSubstitution(False) - # Running always under this own directory. os.chdir(__getProjectRootDir()) - # Running installer in selected mode - runSelectedMode(parser, args) + # Check if installer package is present + if not shutil.which("xmipp3_installer"): + print( + "Xmipp's installer package \'xmipp3-installer\' is not installed.\n" + "It is requied to continue.\n" + "You can install it by running \'pip install xmipp3-installer\'.", + flush=True + ) + sys.exit(1) + + # Call xmipp installer + process = subprocess.run(["xmipp3_installer", *sys.argv[1:]]) + sys.exit(process.returncode) diff --git a/xmipp_old b/xmipp_old deleted file mode 100644 index 5afd22af8c..0000000000 --- a/xmipp_old +++ /dev/null @@ -1,1362 +0,0 @@ -#!/usr/bin/env python3 -# *************************************************************************** -# * Authors: Carlos Oscar S. Sorzano (coss@cnb.csic.es) -# * David Maluenda (dmaluenda@cnb.csic.es) -# * -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# ***************************************************************************/ - - -import os -import re -import shutil -import sys -from datetime import datetime -import json -from scripts import * -from scripts.utils import checkCMakeVersion - - - -# ----K-E-E-P----U-P-D-A-T-E-D---- # -#################################### -XMIPP_VERSION = '3.23.11.0' # -XMIPP_VERNAME = 'devel' # -RELEASE_DATE = '28/11/2023' # -CMAKE_VERSION_REQUIRED = '3.16' # -#################################### - - -XMIPP = 'xmipp' -XMIPP_CORE = 'xmippCore' -XMIPP_VIZ = 'xmippViz' -SCIPION_EM_XMIPP = 'scipion-em-xmipp' -CUFFTADVISOR = 'cuFFTAdvisor' -CTPL = 'CTPL' -GTEST = 'googletest' -LIBSVM = 'libsvm' -LIBCIFPP = 'libcifpp' -VERSION_FILE = 'xmipp_version.txt' -COMPILE_LOG = 'compileLOG.txt' -CONFIG_FILE = 'xmipp.conf' -COMPRESED_REPORT = 'reportInstallation.tar.gz' - -REPOSITORIES = {XMIPP: 'https://github.com/I2PC/xmipp.git', - XMIPP_CORE: 'https://github.com/I2PC/xmippCore.git', - XMIPP_VIZ: 'https://github.com/I2PC/xmippViz.git', - SCIPION_EM_XMIPP: 'https://github.com/I2PC/scipion-em-xmipp.git', - CUFFTADVISOR: 'https://github.com/DStrelak/cuFFTAdvisor.git', - CTPL: 'https://github.com/vit-vit/CTPL.git', - GTEST: 'https://github.com/google/googletest', - LIBSVM: 'https://github.com/cossorzano/libsvm.git', - LIBCIFPP: 'https://github.com/MartinSalinas98/libcifpp'} - -# {dep-repo: (branch, check-conf)} -DEPENDENCIES = {CUFFTADVISOR: ('master', 'CUDA'), - CTPL: ('master', None), - GTEST: ('v1.13.x', None), - LIBSVM: ('master', None), - LIBCIFPP: ('ms_feature_ciflibrary', None)} - -# if a skippable compilation fails (if 'key' found in the failed code), -# a hint is printed in order to export 'value' or edit the config file -SKIPPABLE_BINS = {"optical_alignment": "OPENCV=False", - "volume_homogenizer": "OPENCV=False", - "cuda": "CUDA=False"} - -XMIPP_ENV = {} - - -def checkGithubConnection(): - from http.client import HTTPConnection - from socket import gaierror - conn = HTTPConnection("www.github.com", timeout=3) - try: - conn.request("HEAD", "/") - return True - except gaierror: - return False - finally: - conn.close() - - -def stampVersion(): - LAST_COMPILATION = datetime.now().strftime("%d/%m/%Y") - - def getCommit(repo): - """ In devel mode 'commit.info' should not exist. - In production mode 'commit.info' is added by tar.py - """ - commitFn = os.path.join('src', repo, 'commit.info') - notFoundInfo = "(no git repo detected)" - if os.path.isfile(commitFn): - with open(commitFn, 'r') as file: - commitInfo = file.readline() - elif ensureGit(False)[0]: - found, br, hsh = getCurrentBranch(os.path.join('src', repo), - getHash=True) - commitInfo = "%s (%s)" % (br, hsh) if found else notFoundInfo - else: - commitInfo = notFoundInfo - - return commitInfo - - compilingInfo = {'XMIPP_VERSION': XMIPP_VERSION, - 'RELEASE_DATE': RELEASE_DATE, - 'XMIPP_BRANCH': getCommit(XMIPP), - 'PLUGIN_BRANCH': getCommit(SCIPION_EM_XMIPP), - 'CORE_BRANCH': getCommit(XMIPP_CORE), - 'VIZ_BRANCH': getCommit(XMIPP_VIZ), - 'LAST_COMPILATION': LAST_COMPILATION, - 'XMIPP_VERNAME': XMIPP_VERNAME - } - - versionBinFn = os.path.join('src', 'xmipp', 'applications', 'programs', - 'version', 'version.cpp') - createDir(os.path.dirname(versionBinFn)) - with open(versionBinFn, 'w') as f: - f.write("""// Auto-generated code to get compilation Info -#include -#include -#include -using namespace std; - -int main(int argc, char** argv){ - - if (argc>2) - { - std::cout << "Incorrect parameter" << std::endl; - return 1; - } - int shrt = 0; - if (argc>1) - { - if((strcmp(argv[1], "--short") == 0)) - { - shrt = 1; - }else{ - std::cout << "Incorrect parameter: " << argv[1] << std::endl; - return 2; - } - } - - if (shrt==1) - { - std::cout << "%(XMIPP_VERSION)s" << std::endl; - }else{ - struct utsname utsname; // stores the data returned by uname() - struct utsname *utsname_ptr = &utsname; // pointer to the struct holding the data returned by uname() - - int ret; - ret = uname(utsname_ptr); - - std::cout << std::endl; - std::cout << " \033[4mXmipp version\033[24m: \033[1m%(XMIPP_VERSION)s (%(XMIPP_VERNAME)s)\033[0m" << std::endl; - std::cout << std::endl; - std::cout << " Release date: %(RELEASE_DATE)s" << std::endl; - std::cout << " Xmipp branch: %(XMIPP_BRANCH)s" << std::endl; - std::cout << " Plugin branch: %(PLUGIN_BRANCH)s" << std::endl; - std::cout << " Core branch: %(CORE_BRANCH)s" << std::endl; - std::cout << " Viz branch: %(VIZ_BRANCH)s" << std::endl; - std::cout << " Compilation date: %(LAST_COMPILATION)s" << std::endl; - std::cout << " System compiler: g++ " << __VERSION__ << std::endl; - std::cout << " System: " << utsname.machine << " " << utsname.sysname - << " " << utsname.release << std::endl - << " " << utsname.version << std::endl; - std::cout; - } - return 0; -} -""" % compilingInfo) - -def createDir(dirname): - if not os.path.exists(dirname): - os.makedirs(dirname) - - -def cleanSources(): - for dep in DEPENDENCIES.keys(): - runJob("rm -rf src/%s" % dep) - runJob("rm -rf src/scipion-em-xmipp") - runJob("rm -rf src/xmippCore") - runJob("rm -rf src/xmippViz") - runJob("rm -rf src/xmipp/bin") - runJob("rm -rf src/xmipp/lib") - runJob("rm -rf src/xmipp/.sconsign.dblite") - if ensureGit(False)[0]: - runJob("git stash") # to get exactly like in repo - - -def cleanBinaries(): - for ext in ['so', 'os', 'o']: - runJob('find src/* -name "*.%s" -exec rm -rf {} \;' % ext, showWithReturn=False) - runJob('find . -iname "*.pyc" -delete', showWithReturn=False) - runJob("rm -rf %s build" % Config.FILE_NAME, showWithReturn=False) - # I'm getting ValueError : unsupported pickle protocol: 5' when switching from one python version to another - # This seems to be cached at dblite file. - runJob('find . -iname "*.dblite" -delete', showWithReturn=False) - cleanEmptyFolders() - -def cleanDeprecated(): - listCurrentPrograms = [] - listFilesXmipp = [] - runJob('find src/xmipp/bin/*', log=listFilesXmipp, - show_command=False, show_output=False) - listFilesXmipp = [os.path.basename(x).replace('xmipp_', '') for x in listFilesXmipp] - listCurrentFolders = [x[0] for x in os.walk('src/xmipp/applications/programs/')] - for folder in listCurrentFolders[1:]: - for file in os.listdir(folder): - if '.cpp' in file: - listCurrentPrograms.append(os.path.basename(folder)) - listCurrentScripts = [os.path.basename(x[0]) for x in os.walk('src/xmipp/applications/scripts/')] - listCurrentPrograms = listCurrentPrograms + listCurrentScripts[1:] - list2RemoveXmipp = [x for x in listFilesXmipp if (x not in listCurrentPrograms and 'test_' not in x)] - for x in list2RemoveXmipp: - runJob('rm src/xmipp/bin/xmipp_{}'.format(x)) - if len(list2RemoveXmipp) > 0: - print('Deprecated programs removed') - -def cleanEmptyFolders(): - log = [] - path = "src/xmipp/applications/programs/" - runJob("find {} –type d -empty".format(path), log=log, show_output=False, - showWithReturn=False) - for folder in log: - if os.path.isdir(folder): - runJob("rm -rf {}".format(folder)) - -def checkout(branch): - log = [] - r, currentBranch = getCurrentBranch() - if currentBranch == branch: - return True - isRepositoryClean() - if runJob("git checkout %s" % branch, log=log, show_command=False, show_output=False): - return True - else: - print(yellow("Cannot checkout branch '%s'. Remaining on the branch '%s'." % (branch, currentBranch))) - return True - - -def isRepositoryClean(showError=True): - log = [] - runJob('git diff --name-only', show_output=False, show_command=False, log=log) - if showError and log: # non-empty log means there are uncommited changes - print(red('Repository contains uncommitted changes.')) - print(red("Use 'compileAndInstall' mode to keep developing.")) - return not log - - -def pull(): - log=[] - isRemoteBranch = runJob('git rev-parse HEAD@{upstream}', show_command=False, show_output=False, log=log) - if not isRemoteBranch: - print(yellow(str('Git info: ' + log[0]))) - if checkGithubConnection() and isRemoteBranch: - return runJob("git pull", show_command=False, show_output=False) - print(yellow('Local branch, no upstream for this')) - return True # meaning that this is a local branch or we are offline, so pull doesn't make sense - - -def cloneOrCheckout(repo, branch): - branchName = branch - repo_dir = os.path.join('src', repo) if repo != XMIPP else '.' - if not os.path.exists(repo_dir): - if branch not in getAllBranchesAndTags(repo): - branchName = 'HEAD' - branch = None - print('Cloning repository: {}/{}'.format(repo, branchName)) - print(yellow('Working...'), end='\r') - # If the repo doesn't exist, just clone the whole repo - log = [] - if branch is None: - # let the git client to decide what is the default branch - status = runJob("git clone %s %s" % (REPOSITORIES[repo], repo_dir), - show_output=False, show_command=False, log=log) - if status == False: - exitProgram(1, [False, 1, ''.join(log), "Please review the conexion to the repository"]) - else: - status = runJob("git clone -b %s %s %s" % (branch, REPOSITORIES[repo], repo_dir), - show_output=False, show_command=False, log=log) - if status == False: - exitProgram(1, [False, 1, ''.join(log), "Please review the conexion to the repository"]) - return True - - else: - print('Checkouting and pull repository: {}/{}'.format(repo, branch)) - print(yellow('Working...'), end='\r') - workDir = os.getcwd() - os.chdir(repo_dir) - checkout(branch) - pull() - os.chdir(workDir) - return True - - -def removePrefix(text, prefix): - if text.startswith(prefix): - return text[len(prefix):] - return text - - -def getCurrentCIBranch(): - # see https://docs.travis-ci.com/user/environment-variables/ - # see https://docs.github.com/en/actions/reference/environment-variables#default-environment-variables - # On Travis, PR will have the TRAVIS_PULL_REQUEST_BRANCH variable non-empty - # otherwise the TRAVIS_BRANCH will hold the name of the current branch - # On Github Actions, GITHUB_REF will hold either the tag or branch (including prefix) to the branch which triggered - # the workflow - # For pull request, GITHUB_HEAD_REF will be set (no prefix) - current_branch = ( - os.environ.get('TRAVIS_PULL_REQUEST_BRANCH') or - os.environ.get('TRAVIS_BRANCH') or - os.environ.get('GITHUB_HEAD_REF') or - os.environ.get('GITHUB_REF')) - current_branch = removePrefix(current_branch, 'refs/heads/') - current_branch = removePrefix(current_branch, 'refs/tags/') - print(green("Detected branch: " + current_branch)) - return current_branch is not None, current_branch - - -def getCurrentBranch(cwd='./', getHash=False): - """ If getHash=True: return (success, branch, hash) - If getHash=False: return (success, branch) - """ - outBranchArgs = 2 if getHash else 1 - log = [] - commit = [] - if not os.path.exists(cwd) or not runJob('git rev-parse --short HEAD', - cwd=cwd, show_output=False, - show_command=False, log=commit): - return (False,) + (None,)*outBranchArgs - result = runJob('git name-rev ' + commit[0], cwd=cwd, show_output=False, - show_command=False, log=log) - if result: # log contains commit_space_branchName - return (True,) + tuple(log[0].split()[::-1][0:outBranchArgs]) - print(yellow('Cannot list branches for ' + ''.join(log))) - return (False,) + (None,)*outBranchArgs - - -def getAllBranchesAndTags(repo): - log = [] - result = runJob('git ls-remote -t -h %s' % REPOSITORIES[repo], - show_output=False, log=log, show_command=False) - if result: - branchesAndTags = [l.split('/')[-1] for l in log] - return branchesAndTags - strError = 'Cannot list branches for ' + repo + ' ' + ''.join(log) - status = [False, 1, strError, "Please review the connetion to the repository"] - exitProgram(1, status) - - -def getDefaultBranch(repo): - log = [] - key = 'HEAD branch:' - # this might not work for git < 1.8.5, - # see https://stackoverflow.com/a/32503667/5484355 - # and https://stackoverflow.com/questions/2832269/git-remote-head-is-ambiguous - # In such a case we return None (and during e.g. clone the client should - # decide what is the default branch) - result = runJob('git remote show %s' % REPOSITORIES[repo], - show_output=False, log=log, show_command=False) - if result: - for l in log: - if key in l: - branch = l.split(key)[1] # HEAD branch: devel - return (True, branch.strip()) - exitProgram(1, [False, 1, 'Cannot auto-detect default branch for ' + repo + - '. Maybe git version < 1.8.5?\n' + ''.join(log), - 'Check the git version, the languaje must be english, check also ' - 'the internet connetion']) - - -def getSources(branch): - print("\nGetting Xmipp sources ------------------------------------") - if ensureGit(True)[0] == False: - exitProgram(status[1]) - createDir("src") - repos = [XMIPP_CORE, XMIPP_VIZ, SCIPION_EM_XMIPP] - if not isCIBuild(): - # do not change current commit - repos.append(XMIPP) - for r in repos: - if cloneOrCheckout(r, branch): - print(green('Done ')) - return True - - -def getDependencies(): - print("\nGetting external dependencies ----------------------------") - status = ensureGit(True) - if status[0] == False: - exitProgram(status[1], status) - createDir("src") - for dep, args in DEPENDENCIES.items(): - branch = args[0] - configChecked = not args[1] or buildConfig.is_true(args[1]) - if configChecked: - cloneOrCheckout(dep, branch) - print(green('Done ')) - return True - - -def printVersion(): - buildDir = 'build' - shortFlag = '' - log = [] - for arg in sys.argv[2:]: - if arg == '--short': - shortFlag = '--short' - elif arg.startswith('dir='): - buildDir = arg[4:] - versionBin = "%s/bin/xmipp_version" % buildDir - envSetting = "%s/xmipp.bashrc" % buildDir - if not (os.path.isfile(versionBin) and os.path.isfile(envSetting)): - print("Build not found...") - else: - runJob(". %s ; %s %s" % (envSetting, versionBin, shortFlag), - show_command=False, log=log) - if not log: - log = 'System information not available' - else: - log = '\n'.join(log) - log = log.replace('\x1b[4m', '') - log = log.replace('\x1b[24m', '') - log = log.replace('\x1b[1m', '') - log = log.replace('\x1b[0m', '') - - logCmake = [] - runJob('cmake --version', log=logCmake, show_command=False, show_output=False) - if not logCmake: - logCmake = 'cmake information not available' - else: - logCmake = logCmake[0] - print(' {}'.format(logCmake)) - - logNvcc = [] - runJob('nvcc --version', log=logNvcc, show_command=False, show_output=False) - if not logNvcc: - logNvcc = 'nvcc information not available' - else: - logNvcc = ''.join(logNvcc) - logNvcc = logNvcc[logNvcc.find('release'):logNvcc.find('release') + 12] - print(' nvcc version: {}'.format(logNvcc)) - - with open(VERSION_FILE, 'w') as f: - if log: - f.write(log) - f.write('\n nvcc version: ') - f.write(logNvcc) - f.write('\n ') - f.write(logCmake) - - -def getScipionHome(): - """ Returns SCIPION_HOME, the directory for scipion3 or EMPTY str. """ - return os.environ.get("SCIPION_HOME", whereis("scipion3")) or '' - - -def compileModule(Nproc,module): - sconsProgress = True - printProgress = False - progresLines = 1 - progresLinesPrecompiled = 1 - print(yellow('Working... Visit compileLOG.txt for details'), end='\r') - if module == "xmipp": - write_compileLog('\n\n' + '#' * 27 + '\nCompiling Xmipp------------\n', - COMPILE_LOG=COMPILE_LOG) - printProgress = True - progresLines = 780 #773 - progresLinesPrecompiled = 235 #232 - if module == "xmippCore": - write_compileLog('\n\n' + '#' * 27 + '\nCompiling XmippCore--------\n', - COMPILE_LOG=COMPILE_LOG) - if module == "xmippViz": - write_compileLog('\n\n' + '#' * 27 + '\nCompiling XmippViz---------\n', - COMPILE_LOG=COMPILE_LOG) - log = [] - ok = runJob("/usr/bin/env python3 -u $(which scons) -j%s" % Nproc, "src/%s" % module, log=log, - show_command=False, sconsProgress=sconsProgress, progresLines=progresLines, - progresLinesPrecompiled=progresLinesPrecompiled, printProgress=printProgress, - COMPILE_LOG=COMPILE_LOG) - lastError= '' - #print(yellow('len log scons module: {}: {}'.format(module, len(log)))) - if not ok: - failingBin = None - for l in log[-30:]: # inspecting last 30 lines - # expected error: 'scons: *** [some/program/to/compile] Error 1' - errorRegex = re.match("scons: \*\*\* \[(.*)\] (.*Error) ([0-9]*)[: ]*(.*)", l) - if errorRegex: - failingBin = errorRegex.group(1) - errorType = errorRegex.group(2) - errorNum = errorRegex.group(3) - errorMsg = errorRegex.group(4) - for k, v in SKIPPABLE_BINS.items(): - if k in failingBin: - exitProgram(1, [False, 1, - "\nSome error found compiling '%s' program." - % failingBin.split('/')[-1], - "You can skip this program by including '%s' " - "in the config file." % (v)]) - if 'unsupported pickle protocol' in errorMsg: - exitProgram(1, [False, 1, 'Unsupported pickle protocol', - "\nThis error might be because you changed the python " - "version. If so, please run './xmipp cleanBin' to " - "clean up the installation and, then re-compile Xmipp."]) - errorRegex = re.match(".*: fatal error: (.*): No such file or directory", l) - if errorRegex: - missingPath = errorRegex.group(1) - if lastError != missingPath: - lastError = missingPath - missingDir = missingPath.split(os.path.sep)[0] - missingFile = os.path.join(*missingPath.split(os.path.sep)) - if len(missingFile) > 1: - missingFile = missingFile[1:] - if missingDir in DEPENDENCIES.keys(): - depDir = os.path.join('src', missingDir) - hint = ('cd %s ; git checkout -- %s ; cd -' % (depDir, missingFile) - if os.path.isdir(depDir) else './xmipp get_dependencies') - strSupport = "This file belongs to %s dependency. Please, " \ - "try '%s' and re-compile." % (missingDir, hint) - else: strSupport = 'Run /xmipp cleanAll to remove all repositories ' \ - '(local changes will be removed) and compile Xmipp from scrach' - - exitProgram(1, [False, 1, "\n'%s' file not found." % missingPath, strSupport]) - - exitProgram(1, [False, 1, "Some error occurred during the compilation of %s %s" % - (module, (" ('%s')" % failingBin) if failingBin else ''), - '']) - - print(green('Done. Visit compileLOG.txt for details' + (' ' * 70))) - return ok - - -def compile_cuFFTAdvisor(): - print("Building cuFFTAdvisor...") - print(yellow('Working...'), end='\r') - log = [] - advisorDir = "src/cuFFTAdvisor/" - currDir = os.getcwd() - libDir = "src/xmipp/lib/" - createDir(libDir) - os.chdir(advisorDir) - if runJob("make all", show_command=False, show_output=False, log=log): - log2 = [] - os.chdir(currDir) - if runJob("cp " + advisorDir + "build/libcuFFTAdvisor.so" + " " + libDir, log=log2, show_output=False, show_command=False): - print(green("Done ")) - os.chdir(currDir) - return True, log + log2 - else: - os.chdir(currDir) - print(red(log[0])) - return False, log + log2 - else: - os.chdir(currDir) - print(red(log[0])) - return False, log - - -def compile_googletest(): - print("Building googletest...") - print(yellow('Working...'), end='\r') - log = [] - currDir = os.getcwd() - buildDir = os.path.join("src", "googletest", "build") - createDir(buildDir) - os.chdir(buildDir) - if runJob("cmake ..", show_output=False, show_command=False, log=log): - log2 = [] - if runJob("make gtest gtest_main", show_output=False, - show_command=False, log=log2): - os.chdir(currDir) - print(green("Done ")) - return True, log + log2 - else: - os.chdir(currDir) - print(red(str(log + log2))) - return False, log + log2 - else: - os.chdir(currDir) - print(red(str(log))) - return False, log - - -def compile_libsvm(): - print("Building libsvm...") - print(yellow('Working...'), end='\r') - log = [] - #if the libsvm repo is updated, remember that the repoFork/Makefile was edited to remove references to libsvm-so.2 - currDir = os.getcwd() - libsvmDir = os.path.join("src", "libsvm") - os.chdir(libsvmDir) - if runJob("make lib", show_output=False, show_command=False, log=log): - log2 = [] - libDir = "src/xmipp/lib" - os.chdir(currDir) - createDir(libDir) - if runJob("cp " + libsvmDir + "/libsvm.so" + " " + libDir, log=log2, show_command=False, show_output=False): - os.chdir(currDir) - print(green("Done ")) - return True, log + log2 - else: - os.chdir(currDir) - print(red(log[0])) - return False, log + log2 - else: - os.chdir(currDir) - print(red(log[0])) - return False, log - - -def compile_libcifpp(Nproc): - print("Building libcifpp...") - print(yellow('Working...'), end='\r') - log = [] - # Check if CMake version is up to date - error = checkCMakeVersion(CMAKE_VERSION_REQUIRED) - if error[0] == False: - exitProgram(1, error[1]) - return False, log - currDir = os.getcwd() - # Moving to library directory - libcifppDir = os.path.join("src", "libcifpp") - os.chdir(libcifppDir) - # Installing - fullDir = os.path.join(currDir, libcifppDir, '') - if runJob("cmake -S . -B build -DCMAKE_INSTALL_PREFIX=" + fullDir + " -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCIFPP_DOWNLOAD_CCD=OFF -DCIFPP_INSTALL_UPDATE_SCRIPT=OFF", - show_output=False, show_command=False, log=log): - log1 = [] - if runJob(f"cmake --build build -j {Nproc}", show_output=False, show_command=False, log=log1): - log2 = [] - if runJob("cmake --install build", show_output=False, show_command=False, log=log2): - # Check if libcifpp created up on compiling lib or lib64 directory - libcifppLibDir = "lib64" if os.path.exists("lib64") else "lib" - # Copying .so file - os.chdir(currDir) - libDir = "src/xmipp/lib" - createDir(libDir) - log3 = [] - if runJob("cp " + os.path.join(libcifppDir, libcifppLibDir, "libcifpp.so*") + " " + libDir, log=log3, show_output=False, show_command=False): - print(green("Done" + (' ' * 70))) - return True, log + log1 + log2 + log3 - else: - print(red(log[0])) - return False, log + log1 + log2 + log3 - else: - os.chdir(currDir) - print(red(log[0])) - return False, log + log1 + log2 - else: - os.chdir(currDir) - print(red(log[0])) - return False, log + log1 - else: - os.chdir(currDir) - print(red(log[0])) - return False, log - -def checkScons(): - return runJob("scons --version", show_output=False, show_command=False) - - -def compile(Nproc): - if checkScons(): - if isinstance(Nproc, str) and Nproc.startswith('N='): - Nproc = int(Nproc.split('N=')[1]) - buildConfig.check_version(XMIPP_VERNAME) - todayDate = 'Compilation Date: {}\n'.format(datetime.today()) - write_compileLog(todayDate, append=False, COMPILE_LOG=COMPILE_LOG) - if not compileDependencies(Nproc): - return False - return compileXmipp(Nproc) - else: - exitProgram(15) - -def compileDependencies(Nproc): - print("\nBuilding Dependencies ------------------------------------") - if buildConfig.is_true('CUDA'): - cudaBinDir, nvccBaseName = os.path.split(buildConfig.get()['NVCC']) - # cuFFTAdvisor compilation needs 'nvcc' accessible through the PATH - if not checkProgram(nvccBaseName)[0] and checkProgram(buildConfig.get()['NVCC'])[0]: - # if nvcc basename is not found but absolute path yes, adding the dir to the path. - os.environ['PATH'] = os.pathsep.join([cudaBinDir, os.environ['PATH']]) - if (checkProgram('make') or installDepConda('make')): - write_compileLog('\n\n' + '#' * 27 + '\nCompiling cuFFTAdvisor--------\n', - COMPILE_LOG=COMPILE_LOG) - result, log = compile_cuFFTAdvisor() - write_compileLog('\n'.join(log), COMPILE_LOG=COMPILE_LOG) - if not result: - exitProgram(16) - - write_compileLog('\n\n' + '#' * 27 + '\nCompiling googletest--------\n', - COMPILE_LOG=COMPILE_LOG) - result, log = compile_googletest() - write_compileLog('\n'.join(log), COMPILE_LOG=COMPILE_LOG) - if not result: - exitProgram(17) - - write_compileLog('\n\n' + '#' * 27 + '\nCompiling libsvm--------\n', - COMPILE_LOG=COMPILE_LOG) - result, log = compile_libsvm() - write_compileLog('\n'.join(log), COMPILE_LOG=COMPILE_LOG) - if not result: - exitProgram(18) - - write_compileLog('\n\n' + '#' * 27 + '\nCompiling libcifpp--------\n', - COMPILE_LOG=COMPILE_LOG) - result, log = compile_libcifpp(Nproc) - write_compileLog('\n'.join(log), COMPILE_LOG=COMPILE_LOG) - if not result: - exitProgram(19) - return True - - -def compileXmipp(Nproc): - stampVersion() - print("\nCompiling xmippCore---------------------------------------") - if not compileModule(Nproc,"xmippCore"): - return False - print("\nCompiling xmipp-------------------------------------------") - if not compileModule(Nproc,"xmipp"): - return False - print("\nCompiling xmippViz----------------------------------------") - if not compileModule(Nproc,"xmippViz"): - return False - - # Merge Xmipp and XmippCore compiled databases - cdb_list = ["compile_commands.json", "compile_commands_2.json"] - - merged = list() - for cdb in cdb_list: - if os.path.exists(cdb): - with open(cdb, 'r') as cdb_id: - merged.extend(json.load(cdb_id)) - with open("compile_commands.json", 'w') as cdb_id: - json.dump(merged, cdb_id) - - if os.path.exists("compile_commands_2.json"): - os.remove("compile_commands_2.json") - return True - - -def runTests(testNames): - if len(testNames)==0 or 'help' in testNames or '--help' in testNames: - print("Usage: xmipp test op\n" - "\n" - " op = --show: Show how to invoke all available tests\n" - " --allPrograms: Run all program tests\n" - " --allFuncs: Run all function tests\n" - " 'testName': Run certain test (more than one is available)." - "\n") - return - - print("Testing ---------------------------------------------") - - xmippSrc = os.environ.get('XMIPP_SRC', None) - if xmippSrc and os.path.isdir(xmippSrc): - os.environ['PYTHONPATH'] = ':'.join([ - os.path.join(os.environ['XMIPP_SRC'], XMIPP), - os.environ.get('PYTHONPATH', '')]) - - testsPath = os.path.join(os.environ['XMIPP_SRC'], XMIPP, 'tests') - else: - print(red('XMIPP_SRC is not in the enviroment.') + - '\nTo run the tests you need to run: ' + - blue('source build/xmipp.bashrc')) - exitProgram(0) - - - dataSetPath = os.path.join(testsPath, 'data') - # if not os.path.isdir(dataSetPath): - # createDir(dataSetPath) - os.environ["XMIPP_TEST_DATA"] = dataSetPath - - # downloading/updating the dataset - url = "http://scipion.cnb.csic.es/downloads/scipion/data/tests" - dataset = 'xmipp_programs' - if os.path.isdir(dataSetPath): - print(blue("Updating the test files")) - task = "update" - else: - print(blue("Downloading the test files")) - task = "download" - args = "%s %s %s" % ("tests/data", url, dataset) - runJob("bin/xmipp_sync_data %s %s" % (task, args), cwd='src/xmipp') - - noCudaStr = '--noCuda' if not buildConfig.is_true('CUDA') else '' - print(" Tests to do: %s" % ', '.join(testNames)) - if not runJob("(cd src/xmipp/tests; %s test.py %s %s)" - % (getPython(), ' '.join(testNames), noCudaStr)): - sys.exit(-1) - - -def getPython(): - if checkProgram("scipion3")[0]: - return "scipion3 python" - else: - return 'python3' - - -def addDeepLearninModel(login, modelPath='', update=None): - """ Takes the folder name modelName from models dir and - makes a .tgz, uploads the .tgz to scipion web. - """ - def usageDL(): - print(""" - XMIPP addModel help: - - This mode is used to upload a model folder to the Scipion/Xmipp server. - Usually the model folder contains big files used to fed deep learning procedures - with pretrained data. All the models stored in the server will be downloads - using the 'get_models' mode or during the compilation/installation time - if 'USE_DL=True' in the config file. [or with 'scipion3 installb deepLearningToolkit'] - modelsPath must be the absolute path - Usage: -> ./xmipp addModel [--update] - - Behaviour: 0. modelName = basename(modelsPath) <- Please, check the folder's name! - 1. Packing in 'xmipp_model_modelName.tgz' - 2. Check if that model already exists (use --update to override an existing model) - 3. Upload the model to the server. - 4. Update the MANIFEST file. - - The model name will be the folder name in - """) - sys.exit(0) - - if login == '--help': - usageDL() - - modelPath = modelPath.rstrip("/") - if not os.path.isdir(modelPath): - print(" is not a directory. Please, check the path. \n" - "The name of the model will be the name of that folder.\n") - usageDL() - - modelName = os.path.basename(modelPath) - modelsDir = os.path.dirname(modelPath) - tgzFn = "xmipp_model_%s.tgz" % modelName - localFn = os.path.join(modelsDir, tgzFn) - - print("Creating the '%s' model." % tgzFn) - runJob("tar czf %s %s" % (tgzFn, modelName), cwd=modelsDir) - - remotePath = "scipionfiles/downloads/scipion/software/em" - print("Warning: Uploading, please BE CAREFUL! This can be dangerous.") - print('You are going to be connected to "%s" to write in folder ' - '"%s".' % (login, remotePath)) - if input("Continue? YES/no\n").lower() == 'no': - sys.exit() - - print("Trying to upload the model using '%s' as login" % login) - args = "%s %s %s %s" % (login, os.path.abspath(localFn), remotePath, update) - if runJob("src/xmipp/bin/xmipp_sync_data upload %s" % args): - print("'%s' model successfully uploaded! Removing the local .tgz" - % modelName) - runJob("rm %s" % localFn) - - -pDLdownload = None -def downloadDeepLearningModels(dest, dedicatedMode=False): - if not buildConfig.is_true('USE_DL') and not dedicatedMode: - return True - url = "http://scipion.cnb.csic.es/downloads/scipion/software/em" - - if not os.path.exists('build/bin/xmipp_sync_data'): - print(red('Xmipp has not been installed. Please, first install it ')) - return False - if dest == 'build': - modelsPath = os.path.join(dest, 'models') - else: - modelsPath = dest - dataSet = "DLmodels" - - # downloading/updating the DLmodels - if os.path.isdir(modelsPath): - print("Updating the Deep Learning models (in backgound)") - task = "update" - else: - print("Downloading Deep Learning models (in backgound)") - task = "download" - global pDLdownload - - # using Popen instead of runJob in order to download in parallel - pDLdownload = runJob("bin/xmipp_sync_data %s %s %s %s" - % (task, dest, url, dataSet), - cwd='build', show_command=False, - in_parallel=not dedicatedMode) - if dedicatedMode: - ok = pDLdownload - else: # in parallel poll() is None untill finished - ok = pDLdownload.poll() is None or pDLdownload.poll() == 0 - return ok - - -def linkToScipion(dirname): - scipionSoftware = os.environ.get('SCIPION_SOFTWARE', - os.path.join(getScipionHome(), 'software')) - scipionLibs = os.path.join(scipionSoftware, 'lib') - scipionBindings = os.path.join(scipionSoftware, 'bindings') - scipionSoftwareEM = os.path.join(scipionSoftware, 'em') - xmippHomeLink = os.path.join(scipionSoftwareEM, 'xmipp') - currentDir = os.getcwd() - dirnameAbs = os.path.join(currentDir,dirname) - if os.path.isdir(scipionLibs) and os.path.isdir(scipionBindings): - print("\nLinking to Scipion ---------------------------------------") - print(yellow('Working...'), end='\r') - if os.path.isdir(xmippHomeLink): - runJob("rm %s" %xmippHomeLink) - runJob("ln -srf %s %s" % (dirnameAbs, xmippHomeLink)) - xmippLink = os.readlink(xmippHomeLink) - coreLib = os.path.join(xmippLink, "lib", "libXmippCore.so") - xmippLib = os.path.join(xmippLink, "lib", "libXmipp.so") - SVMLib = os.path.join(xmippLink, "lib", "libsvm.so") - CIFPPLib = os.path.join(xmippLink, "lib", "libcifpp.so*") - bindings = os.path.join(xmippLink, "bindings", "python", "*") - - os.chdir(scipionSoftwareEM) - runJob("ln -srf %s %s" % (coreLib, scipionLibs)) - runJob("ln -srf %s %s" % (SVMLib, scipionLibs)) - runJob("ln -srf %s %s" % (CIFPPLib, scipionLibs)) - runJob("ln -srf %s %s" % (xmippLib, scipionLibs)) - runJob("ln -srf %s %s" % (bindings, scipionBindings)) - os.chdir(currentDir) - print(green(str("Xmipp linked to Scipion on " + xmippHomeLink) + (' ' * 150))) - - - else: - print(yellow("No scipion3 found. If you intended to use Xmipp in " - "the Scipion framework:\ncompile Xmipp " - "with Scipion './scipion3 run ./xmipp' or check the binding at " - "SCIPION_HOME/software/bindings...")) - - -def install(dirname): - print("\nInstalling -----------------------------------------------") - print(yellow('Working...'), end='\r') - if XMIPP_VERNAME =='devel': show_command = True - else: show_command = False - cleanDeprecated() - cpCmd = "rsync -LptgoD " if checkProgram("rsync") else "cp" - ok = True - - createDir(dirname) - - createDir(dirname+"/lib") - ok = ok and runJob(cpCmd+" src/*/lib/lib* "+dirname+"/lib/", show_command=show_command) - - - if os.path.exists(dirname + "/bin"): - shutil.rmtree(dirname + "/bin") - createDir(dirname+"/bin") - dirBin = os.path.join(os.getcwd(), "src/xmipp/bin/") - filenames = os.listdir(dirBin) - - for f in filenames: - if os.path.islink(os.path.join(dirBin, f)): - ok = ok and runJob('ln -s ' + os.path.join(dirBin, f) + ' ' + os.path.join(dirname, 'bin', f), show_command=show_command, show_output=False) - else: - ok = ok and runJob(cpCmd + os.path.join(dirBin, f) + ' ' + os.path.join(dirname, 'bin', f), show_command=show_command) - - destPathPyModule = os.path.expanduser(os.path.abspath(os.path.join(dirname, "pylib", "xmippPyModules"))) - createDir(destPathPyModule) - initFn = destPathPyModule + "/__init__.py" - if not os.path.isfile(initFn): - with open(initFn, 'w') as f: - pass # just to create a init file to be able to import it as module - - - dirBin = os.path.join(os.getcwd(), 'src/xmipp/libraries/py_xmipp') - folders = list(os.walk(dirBin)) - for folderPath, _, folderFiles in folders: - folderName = os.path.relpath(folderPath, dirBin) - createDir(os.path.join(destPathPyModule, folderName)) - for file in folderFiles: - runJob("ln -s " + os.path.join(folderPath, file) + ' ' + os.path.join(destPathPyModule, folderName, file), - show_output=False, show_command=show_command, log=[]) - - - createDir(dirname+"/bindings") - createDir(dirname+"/bindings/matlab") - ok = ok and runJob(cpCmd+" src/xmipp/bindings/matlab/*.m* "+dirname+"/bindings/matlab/", show_command=show_command) - - createDir(dirname+"/bindings/python") - ok = ok and runJob(cpCmd+" src/xmipp/bindings/python/xmipp_base.py "+dirname+"/bindings/python/", show_command=show_command) - ok = ok and runJob(cpCmd+" src/xmipp/bindings/python/xmipp.py " + dirname + "/bindings/python/", show_command=show_command) - ok = ok and runJob(cpCmd+" src/xmipp/bindings/python/xmipp_conda_envs.py " + dirname + "/bindings/python/", show_command=show_command) - ok = ok and runJob(cpCmd+" -r src/xmipp/bindings/python/envs_DLTK/ " + dirname + "/bindings/python/envs_DLTK", show_command=show_command) - ok = ok and runJob(cpCmd+" src/xmipp/lib/xmippLib.so "+dirname+"/bindings/python/", show_command=show_command) - ok = ok and runJob(cpCmd+" src/xmipp/lib/_swig_frm.so "+dirname+"/bindings/python/", show_command=show_command) - - createDir(dirname+"/bindings/python/sh_alignment") - ok = ok and runJob(cpCmd+" -r src/xmipp/external/sh_alignment/python/* "+dirname+"/bindings/python/sh_alignment/", show_command=show_command) - ok = ok and runJob(cpCmd+" src/xmipp/external/sh_alignment/swig_frm.py "+dirname+"/bindings/python/sh_alignment/", show_command=show_command) - - createDir(dirname+"/resources") - ok = ok and runJob(cpCmd+" -r src/*/resources/* "+dirname+"/resources/", show_command=show_command) - - # ok = ok and runJob(cpCmd + " -r src/xmippViz/bindings/chimera " + dirname + "/bindings/") - createDir(dirname+"/bindings/java") - ok = ok and runJob(cpCmd+" -Lr src/xmippViz/java/lib "+dirname+"/bindings/java/", show_command=show_command) - ok = ok and runJob(cpCmd+" -Lr src/xmippViz/java/build "+dirname+"/bindings/java/", show_command=show_command) - ok = ok and runJob(cpCmd+" -Lr src/xmippViz/external/imagej "+dirname+"/bindings/java/", show_command=show_command) - ok = ok and runJob(cpCmd+" src/xmippViz/bindings/python/xmippViz.py "+dirname+"/bindings/python/", show_command=show_command) - - ok = ok and runJob(cpCmd+" xmippEnv.json "+dirname+"/xmippEnv.json", show_command=show_command) - - if not ok: - print(red("\nSome error occurred during the installation.\n")) - exitProgram() - return False - print(green('Xmipp installed on {}'.format(os.path.join(os.getcwd(), dirname)))) - - # Scipion connection - linkToScipion(dirname) - - runJob("touch %s/v%s" % (dirname, XMIPP_VERSION), show_command=False) # version token - fhBash = open(dirname+"/xmipp.bashrc","w") - fhFish = open(dirname+"/xmipp.fish","w") - fhBash.write("# This script is valid for bash and zsh\n\n") - fhFish.write("# This script is valid for fish\n\n") - - XMIPP_HOME = os.path.realpath(dirname) - fhBash.write("export XMIPP_HOME=%s\n"%XMIPP_HOME) - fhFish.write("set -x XMIPP_HOME %s\n"%XMIPP_HOME) - - XMIPP_SRC = os.path.realpath("src") - fhBash.write("export XMIPP_SRC=%s\n"%XMIPP_SRC) - fhFish.write("set -x XMIPP_SRC %s\n"%XMIPP_SRC) - - # SCIPION_HOME = getScipionHome() - # if SCIPION_HOME: - # fhBash.write("export PATH=$SCIPION_HOME/software/bin:$PATH\n") - # fhBash.write("export LD_LIBRARY_PATH=$SCIPION_HOME/software/lib:$LD_LIBRARY_PATH\n") - # #fhFish.write("set -px PATH $SCIPION_HOME/software/bin\n") - # fhFish.write("set -px LD_LIBRARY_PATH $SCIPION_HOME/software/lib\n") - virtEnvDir = os.environ.get('VIRTUAL_ENV', '') # if virtualEnv is used - virtEnvLib = os.path.join(virtEnvDir, 'lib') if virtEnvDir else '' - condaDir = os.environ.get('CONDA_PREFIX', '') # if conda is used - condaLib = os.path.join(condaDir, 'lib') if condaDir else '' - fhBash.write("export PATH=%s/bin:$PATH\n"%XMIPP_HOME) - fhBash.write("export LD_LIBRARY_PATH=%s/lib:%s/bindings/python:%s:%s:$LD_LIBRARY_PATH\n" - %(XMIPP_HOME, XMIPP_HOME, virtEnvLib, condaLib)) - fhBash.write("export PYTHONPATH=%s/bindings/python:%s/pylib:$PYTHONPATH\n"%(XMIPP_HOME,XMIPP_HOME)) - fhFish.write("set -px PATH %s/bin\n"%XMIPP_HOME) - fhFish.write("set -px LD_LIBRARY_PATH %s/lib %s/bindings/python %s %s\n" - %(XMIPP_HOME, XMIPP_HOME, virtEnvLib, condaLib)) - fhFish.write("set -px PYTHONPATH %s/bindings %s/pylib\n"%(XMIPP_HOME,XMIPP_HOME)) - - fhBash.write('\n') - fhBash.write("alias x='xmipp'\n") - fhBash.write("alias xsj='xmipp_showj'\n") - fhBash.write("alias xio='xmipp_image_operate'\n") - fhBash.write("alias xis='xmipp_image_statistics'\n") - fhBash.write("alias xih='xmipp_image_header'\n") - fhBash.write("alias xmu='xmipp_metadata_utilities'\n") - fhFish.write('\n') - fhFish.write("alias x 'xmipp'\n") - fhFish.write("alias xsj 'xmipp_showj'\n") - fhFish.write("alias xio 'xmipp_image_operate'\n") - fhFish.write("alias xis 'xmipp_image_statistics'\n") - fhFish.write("alias xih 'xmipp_image_header'\n") - fhFish.write("alias xmu 'xmipp_metadata_utilities'\n") - - fhBash.close() - fhFish.close() - - endMessage(XMIPP_VERNAME) - return True - - -def writeDevelPaths(dirname): - fhBash = open(dirname+"/xmipp.bashrc","w") - - XMIPP_HOME = os.path.realpath(dirname) - fhBash.write("export XMIPP_HOME=%s\n"%XMIPP_HOME) - - XMIPP_SRC = os.path.realpath("src") - fhBash.write("export XMIPP_SRC=%s\n"%XMIPP_SRC) - - # SCIPION_HOME = getScipionHome() - # if SCIPION_HOME: - # fhBash.write("export PATH=$SCIPION_HOME/bin:$PATH\n") - # fhBash.write("export LD_LIBRARY_PATH=$SCIPION_HOME/software/lib:$LD_LIBRARY_PATH\n") - - fhBash.write("export PATH=%s/xmipp/bin:%s/xmippViz/bin:$PATH\n"%(XMIPP_HOME,XMIPP_HOME)) - fhBash.write("export LD_LIBRARY_PATH=%s/xmippCore/lib:$LD_LIBRARY_PATH\n"%XMIPP_HOME) - fhBash.write("export LD_LIBRARY_PATH=%s/xmippCore/bindings/python:$LD_LIBRARY_PATH\n"%XMIPP_HOME) - fhBash.write("export LD_LIBRARY_PATH=%s/xmipp/lib:$LD_LIBRARY_PATH\n"%XMIPP_HOME) - fhBash.write("export LD_LIBRARY_PATH=%s/xmipp/bindings/python:$LD_LIBRARY_PATH\n"%XMIPP_HOME) - fhBash.write("export PYTHONPATH=%s/xmippCore/bindings/python:$PYTHONPATH\n"%XMIPP_HOME) - fhBash.write("export PYTHONPATH=%s/xmipp/bindings/python:$PYTHONPATH\n"%XMIPP_HOME) - fhBash.write("export PYTHONPATH=%s/xmippViz/bindings/python:$PYTHONPATH\n"%XMIPP_HOME) - - fhBash.close() - - -def usage(msg=''): - if msg != '': - print(red(msg)) - print("Usage: xmipp [options]\n" - " ----------------------------\n" - " version [dir=build] Returns the version information. Add '--short' to print only the version number.\n" - " compile [N] Compile with N processors (8 by default)\n" - " install [dir] Install at dir (./build by default)\n" - " compileAndInstall [N] [dir] Compile with N processors (8 by default) and install in the dir directory ('build' by\n" - " default)\n" - " all [op1=opt1 op2=opt2...] (Default) Retrieve [br=branch], configure, check, compile [N=8], install [dir=build]\n" - " ----------------------------\n" - " config [noAsk] Configure compilation variables. If 'noAsk' is passed, it will try to automatically \n" - " found some libraries and compilers. \n" - " for compiling using system libraries\n" - " check_config Check that the configuration is correct\n" - " ----------------------------\n" - " get_dependencies Retrieve dependencies from github\n" - " get_devel_sources [branch] Retrieve development sources from github for a given branch (devel branch by default)\n" - " get_models [dir] Download the Deep Learning Models at dir/models (./build/models by default).\n" - " ----------------------------\n" - " cleanBin Clean all already compiled files (build, .so,.os,.o in src/* and " + Config.FILE_NAME + ")\n" - " cleanDeprecated Clean all deprecated executables from src/xmipp/bin).\n" - " cleanAll Delete all (sources and build directories)\n" - " ----------------------------\n" - " test [--show] testName: Run tests to check Xmipp programs (without args, it shows a detailed help).\n" - " if --show is activated without testName all are shown, \n" - " instead a grep of testName is done \n" - " ----------------------------\n" - " For developers:\n" - " create_devel_paths Create bashrc files for devel\n" - " git [command] Git command to all 4 repositories\n" - " gitConfig Change the git config from https to git\n" - " addModel login modelPath Takes a deepLearning model from the 'modelPath', makes a tgz of it and \n" - " uploads the .tgz according to the . \n" - " Note the login (usr@server) must have write permisions to Nolan machine.\n" - " tar [v=ver] [br=br] Create a bundle of the xmipp (without arguments shows a detailed help)\n" - " can be 'Sources', 'BinDebian' or 'BinCentos', when Sources \n" - " put a branch (default: master).'\n" - " usually X.YY.MM (add debug to package this local script and \n" - " the local scripts/tar.py) \n" - ) - - -def exitProgram(error=0, status=None): - try: - log = [] - runJob( - 'tar -czf {} {} {} {} '.format(COMPRESED_REPORT, - CONFIG_FILE, - VERSION_FILE, - COMPILE_LOG), - show_output=False, show_command=False, log=log) - except Exception as e: - pass - if error: - errorEndMessage(XMIPP_VERNAME, error, status) - printVersion() - sys.exit(1) - sys.exit(0) - - -if __name__ == '__main__': - # Running always under this own directory. - os.chdir(os.path.dirname(os.path.abspath(__file__))) - askUser = True - - if 'noAsk' in sys.argv: - askUser = False - sys.argv.pop(sys.argv.index('noAsk')) - - n = len(sys.argv) - if n == 2 and (sys.argv[1]=="help" or sys.argv[1]=="-help" or sys.argv[1]=="--help" or sys.argv[1]=="-h"): - usage() - sys.exit(0) - for idx, arg in enumerate(sys.argv): - if ' ' in arg: # to preserve spaces between "comas" - sys.argv[idx] = '"%s"' % sys.argv[idx] - if n >= 2: - mode = sys.argv[1] - else: - mode = "all" - - Config - buildConfig = Config(askUser, ) - buildConfig.read() - - if mode == "cleanAll": - print("WARNING: This will DELETE ALL content from src and build") - print(" Notice that if you have unpushed changes, \n" - " they will be deleted.\n") - print("Are you sure you want to do this? (YeS/No) -case sensitive-") - yesno = input() - if yesno == "YeS": - print("Cleaning everything") - cleanSources() - cleanBinaries() - else: - print("Nothing cleaned") - if yesno.lower()=="yes": - print("Pay attention to capital letters of YeS") - elif mode == "get_dependencies": - getDependencies() - elif mode == "cleanBin": - cleanBinaries() - elif mode == 'cleanDeprecated': - cleanDeprecated() - elif mode == "version": - printVersion() - elif mode == "get_devel_sources": - branch = None if n==2 else sys.argv[2] - getSources(branch) - elif mode == "config": - buildConfig.create() - elif mode == "check_config": - if not buildConfig.check(): - print(red("\nCheck failed! Something wrong with the configuration.\n")) - sys.exit(1) - elif mode == "compile": - Nproc = 8 if n < 3 else sys.argv[2] - ok = compile(Nproc) - module = 'Xmipp' - if ok: - print("\n" - " * %s has been successfully compiled * \n" - " > > > Don't forget to install! < < < \n\n" - % module) - exitProgram(error=0) - else: - print(red("\nSome error occurred during the compilation\n")) - exitProgram(error=1) - elif mode == "compileAndInstall": - Nproc = 8 - dir = "build" - if n > 2: - for arg in sys.argv[2:]: - if arg.isdigit() or arg.startswith('N='): - Nproc = arg - else: - dir = arg - removeCompileAndReportFile(COMPRESED_REPORT, COMPILE_LOG) - ok = compile(Nproc) - ok = ok and install(dir) - if not ok: - exitProgram(error=1) - else: - exitProgram(error=0) - - elif mode == "install": - if n == 3: - dir = sys.argv[2] - else: - dir = "build" - ok = install(dir) - if not ok: - sys.exit(1) - elif mode == "get_models": - modelsDir = 'build' if n==2 else sys.argv[2] - downloadDeepLearningModels(modelsDir, dedicatedMode=True) - elif mode == "test" or mode == "tests": - runTests(sys.argv[2:]) - elif mode == "all": - Nproc = 8 - branch = '' - buildDir = 'build' - for arg in sys.argv[2:]: - if arg.startswith("N="): - Nproc = int(arg[2:]) - elif arg.startswith("br="): - branch = arg[3:] - elif arg.startswith("dir="): - buildDir = arg[4:] - else: - usage("Unknown %s argument"%arg) - sys.exit(1) - # create config if not there - if not os.path.isfile(Config.FILE_NAME): - buildConfig.create() - else: - print(green("'%s' detected." % Config.FILE_NAME)) - # HACK: re-read it from file to resolve paths - removeCompileAndReportFile(COMPRESED_REPORT, COMPILE_LOG) - buildConfig.read() - status = buildConfig.check() - if status[0] == False: #status = [False, index, error msg, suport Msg] - if len(status) > 2: - exitProgram(status[1], status) - else: - exitProgram(status[1]) - ok = downloadDeepLearningModels(buildDir) - ok = ok and getDependencies()\ - and getSources(branch)\ - and compile(Nproc)\ - and install(buildDir) - if ok: - exitProgram() - else: - exitProgram(error=1) - elif mode == "create_devel_paths": - if n == 3: - dir = sys.argv[2] - else: - dir = "." - writeDevelPaths(dir) - elif mode == "git": - status = ensureGit(True) - if status [0] == False: - exitProgram(status[1]) - print('-Repository xmippCore:\t\t', end='') - runJob("(cd src/xmippCore; git %s)"%" ".join(sys.argv[2:]), show_command=False) - print('\n-Repository xmipp:\t\t', end='') - runJob("(cd src/xmipp; git %s)"%" ".join(sys.argv[2:]), show_command=False) - print('\n-Repository xmippViz:\t\t', end='') - runJob("(cd src/xmippViz; git %s)"%" ".join(sys.argv[2:]), show_command=False) - print('\n-Repository scipion-em-xmipp:\t', end='') - runJob("(cd src/scipion-em-xmipp; git %s)"%" ".join(sys.argv[2:]), show_command=False) - elif mode == "gitConfig": - status = ensureGit(True) - if status [0] == False: - exitProgram(status[1]) - runJob("sed -i 's/https:\/\/github.com\//git@github.com:/g' src/xmippCore/.git/config", showWithReturn=False) - runJob("sed -i 's/https:\/\/github.com\//git@github.com:/g' .git/config", showWithReturn=False) - runJob("sed -i 's/https:\/\/github.com\//git@github.com:/g' src/xmippViz/.git/config", showWithReturn=False) - runJob("sed -i 's/https:\/\/github.com\//git@github.com:/g' src/scipion-em-xmipp/.git/config", showWithReturn=False) - elif mode == 'addModel': - update = False - if not (n == 4 or (n == 5 and sys.argv[4] == '--update') or (n == 3 and sys.argv[2] == '--help')): - print("Incorrect number of parameters.\n") - usage() - sys.exit(1) - addDeepLearninModel(*sys.argv[2:]) - elif mode == 'tar': - if len(sys.argv) < 3: - runJob("scripts/tar.py --help") - sys.exit(0) - ver = XMIPP_VERSION - br = 'master' - mode = sys.argv[2] - debugFlag = '' - for arg in sys.argv[3:]: - if arg.startswith('br='): - br = arg.split('br=')[1] - if arg.startswith('v='): - ver = arg.split('v=')[1] - if arg.lower() == 'debug': - debugFlag = 'debug' - runJob("scripts/tar.py %s %s %s %s" % (mode, ver, br, debugFlag)) - else: - usage(" -> option not found <- \n") \ No newline at end of file