diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 66158b413688e..c3ee8e4148ecd 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -41,7 +41,7 @@ jobs: export CI=azure export CONFIG=linux64 - export DOCKER_IMAGE=quay.io/condaforge/linux-anvil-cos7-x86_64 + export DOCKER_IMAGE=quay.io/condaforge/linux-anvil-x86_64:alma9 export AZURE=True .scripts/run_docker_build.sh @@ -54,7 +54,7 @@ jobs: - publish: build_artifacts/noarch/ artifact: conda_pkgs_noarch -- job: linux_64_cuda_112 +- job: linux_64_cuda_118 dependsOn: linux_64 condition: and(not(eq(variables['Build.SourceBranch'], 'refs/heads/main')), eq(dependencies.linux_64.outputs['linux_64_build.NEED_CUDA'], '1')) pool: @@ -89,16 +89,16 @@ jobs: mkdir -p build_artifacts/linux-64/ export CI=azure - export CONFIG=linux64_cuda112 - export DOCKER_IMAGE=quay.io/condaforge/linux-anvil-cuda:11.2 + export CONFIG=linux64_cuda118 + export DOCKER_IMAGE=quay.io/condaforge/linux-anvil-x86_64-cuda11.8:ubi8 export AZURE=True .scripts/run_docker_build.sh - displayName: Run docker build for CUDA 11.2 + displayName: Run docker build for CUDA 11.8 - publish: build_artifacts/linux-64/ - artifact: conda_pkgs_linux_64_cuda112 + artifact: conda_pkgs_linux_64_cuda118 -- job: linux_64_cuda_118 +- job: linux_64_cuda_120 dependsOn: linux_64 condition: and(not(eq(variables['Build.SourceBranch'], 'refs/heads/main')), eq(dependencies.linux_64.outputs['linux_64_build.NEED_CUDA'], '1')) pool: @@ -133,11 +133,11 @@ jobs: mkdir -p build_artifacts/linux-64/ export CI=azure - export CONFIG=linux64_cuda118 - export DOCKER_IMAGE=quay.io/condaforge/linux-anvil-cuda:11.8 + export CONFIG=linux64_cuda120 + export DOCKER_IMAGE=quay.io/condaforge/linux-anvil-x86_64:alma9 export AZURE=True .scripts/run_docker_build.sh - displayName: Run docker build for CUDA 11.8 + displayName: Run docker build for CUDA 12.0 - publish: build_artifacts/linux-64/ - artifact: conda_pkgs_linux_64_cuda118 + artifact: conda_pkgs_linux_64_cuda120 diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml index da225d13d3fe9..2411eeff46f2b 100755 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -6,7 +6,7 @@ jobs: - job: osx condition: not(eq(variables['Build.SourceBranch'], 'refs/heads/main')) pool: - vmImage: macOS-11 + vmImage: macOS-13 strategy: matrix: osx_64: @@ -16,9 +16,11 @@ jobs: steps: - script: | - export CI=azure ./.scripts/run_osx_build.sh displayName: Run OSX build + env: + CI: azure + CONDA_BLD_PATH: /Users/runner/bld - - publish: /Users/runner/Miniforge3/conda-bld/osx-64/ + - publish: /Users/runner/bld/osx-64/ artifact: conda_pkgs_osx diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 916e1ea6a4aaf..9b0771485eb57 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -10,27 +10,13 @@ jobs: CONFIG: win64 timeoutInMinutes: 360 steps: - - task: PythonScript@0 - displayName: 'Download Miniforge' - inputs: - scriptSource: inline - script: | - import urllib.request - url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe' - path = r"$(Build.ArtifactStagingDirectory)/Miniforge.exe" - urllib.request.urlretrieve(url, path) - - script: | - start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\Miniforge - displayName: Install Miniforge - - powershell: Write-Host "##vso[task.prependpath]C:\Miniforge\Scripts" - displayName: Add conda to PATH - - script: | call .scripts\run_win_build.bat displayName: Build recipes env: CI: azure - CONDA_BLD_PATH: C:\bld + CONDA_BLD_PATH: D:\bld + MINIFORGE_HOME: D:\Miniforge - - publish: C:\\bld\\win-64\\ + - publish: D:\\bld\\win-64\\ artifact: conda_pkgs_win diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index b75681ef4666c..c0613967b3a25 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -1,11 +1,18 @@ -import conda_build.conda_interface -import networkx as nx +from shutil import rmtree +import tempfile +import conda.base.context +import conda.core.index +import conda.resolve import conda_build.api import conda_index.api +import networkx as nx from compute_build_graph import construct_graph import argparse +import atexit +import re import os from collections import OrderedDict +from pathlib import Path import sys import subprocess import yaml @@ -16,6 +23,9 @@ from yaml import BaseLoader, load +EXAMPLE_RECIPE_FOLDERS = ["example", "example-v1"] +LOCAL_CHANNELS = os.environ.get("CONDA_BLD_PATH", "local").split(",") + def get_host_platform(): from sys import platform if platform == "linux" or platform == "linux2": @@ -32,7 +42,12 @@ def get_config_name(arch): def build_all(recipes_dir, arch): - folders = list(filter(lambda d: os.path.isdir(os.path.join(recipes_dir, d)), os.listdir(recipes_dir))) + folders = [ + d + for d in os.listdir(recipes_dir) + if os.path.isdir(os.path.join(recipes_dir, d)) + and d not in EXAMPLE_RECIPE_FOLDERS + ] if not folders: print("Found no recipes to build") return @@ -41,21 +56,59 @@ def build_all(recipes_dir, arch): script_dir = os.path.dirname(os.path.realpath(__file__)) variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch))) + has_meta_yaml = False + has_recipe_yaml = False + found_cuda = False found_centos7 = False for folder in folders: meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml") if os.path.exists(meta_yaml): + has_meta_yaml = True with(open(meta_yaml, "r", encoding="utf-8")) as f: text = ''.join(f.readlines()) if 'cuda' in text: found_cuda = True if 'sysroot_linux-64' in text: found_centos7 = True + + recipe_yaml = os.path.join(recipes_dir, folder, "recipe.yaml") + if os.path.exists(recipe_yaml): + has_recipe_yaml = True + with open(recipe_yaml, "r", encoding="utf-8") as f: + text = "".join(f.readlines()) + if "cuda" in text: + found_cuda = True + if "sysroot_linux-64" in text: + found_centos7 = True + + cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml") + if os.path.exists(cbc): + with open(cbc, "r") as f: + lines = f.readlines() + pat = re.compile(r"^([^\#]*?)\s+\#\s\[.*(not\s(linux|unix)|(?/" + ) def read_mambabuild(recipes_dir): @@ -196,7 +320,7 @@ def read_mambabuild(recipes_dir): folders = os.listdir(recipes_dir) conda_build_tools = [] for folder in folders: - if folder == "example": + if folder in EXAMPLE_RECIPE_FOLDERS: continue cf = os.path.join(recipes_dir, folder, "conda-forge.yml") if os.path.exists(cf): @@ -217,8 +341,11 @@ def use_mambabuild(): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--arch', default='64', - help='target architecture (64 or 32)') + parser.add_argument( + '--arch', + default='64', + help='target architecture (second component of a subdir; e.g. 64, arm64, ppc64le)' + ) args = parser.parse_args() root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) check_recipes_in_correct_dir(root_dir, "recipes") diff --git a/.ci_support/compute_build_graph.py b/.ci_support/compute_build_graph.py index 6789d337f3609..0fcb04a108eb7 100644 --- a/.ci_support/compute_build_graph.py +++ b/.ci_support/compute_build_graph.py @@ -36,19 +36,36 @@ import pkg_resources import re import subprocess +import functools +from functools import lru_cache +from frozendict import frozendict, deepfreeze import networkx as nx -from conda_build import api, conda_interface +from conda.models.match_spec import MatchSpec +from conda.models.records import PackageRecord +from conda_build import api from conda_build.metadata import find_recipe, MetaData -from conda_build.utils import HashableDict - log = logging.getLogger(__file__) CONDA_BUILD_CACHE = os.environ.get("CONDA_BUILD_CACHE") hash_length = api.Config().hash_length +# https://stackoverflow.com/questions/6358481/using-functools-lru-cache-with-dictionary-arguments +def freezeargs(func): + """Convert a mutable dictionary into immutable. + Useful to be compatible with cache + """ + + @functools.wraps(func) + def wrapped(*args, **kwargs): + args = (frozendict(arg) if isinstance(arg, dict) else arg for arg in args) + kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} + return func(*args, **kwargs) + return wrapped + + def package_key(metadata, worker_label, run='build'): # get the build string from whatever conda-build makes of the configuration used_loop_vars = metadata.get_used_loop_vars() @@ -199,7 +216,8 @@ def get_run_test_deps(meta): _rendered_recipes = {} -@conda_interface.memoized +@freezeargs +@lru_cache(maxsize=None) def _get_or_render_metadata(meta_file_or_recipe_dir, worker, finalize, config=None): global _rendered_recipes platform = worker['platform'] @@ -216,6 +234,7 @@ def _get_or_render_metadata(meta_file_or_recipe_dir, worker, finalize, config=No def add_recipe_to_graph(recipe_dir, graph, run, worker, conda_resolve, recipes_dir=None, config=None, finalize=False): try: + print(recipe_dir, worker, config, finalize, flush=True) rendered = _get_or_render_metadata(recipe_dir, worker, config=config, finalize=finalize) except (IOError, SystemExit) as e: log.exception('invalid recipe dir: %s', recipe_dir) @@ -253,15 +272,14 @@ def match_peer_job(target_matchspec, other_m, this_m=None): match_dict = {'name': other_m.name(), 'version': other_m.version(), 'build': _fix_any(other_m.build_id(), other_m.config), } - match_dict = conda_interface.Dist(name=match_dict['name'], - dist_name='-'.join((match_dict['name'], - match_dict['version'], - match_dict['build'])), - version=match_dict['version'], - build_string=match_dict['build'], - build_number=int(other_m.build_number() or 0), - channel=None) - matchspec_matches = target_matchspec.match(match_dict) + match_record = PackageRecord( + name=match_dict['name'], + version=match_dict['version'], + build=match_dict['build'], + build_number=int(other_m.build_number() or 0), + channel=None, + ) + matchspec_matches = target_matchspec.match(match_record) variant_matches = True if this_m: @@ -294,13 +312,13 @@ def add_intradependencies(graph): log.info(" test: {}".format(test_requires)) deps = set(m.ms_depends('build') + m.ms_depends('host') + m.ms_depends('run') + - [conda_interface.MatchSpec(dep) for dep in test_requires or []]) + [MatchSpec(dep) for dep in test_requires or []]) for dep in deps: name_matches = (n for n in graph.nodes() if graph.nodes[n]['meta'].name() == dep.name) for matching_node in name_matches: # are any of these build dependencies also nodes in our graph? - if (match_peer_job(conda_interface.MatchSpec(dep), + if (match_peer_job(MatchSpec(dep), graph.nodes[matching_node]['meta'], m) and (node, matching_node) not in graph.edges()): @@ -327,7 +345,7 @@ def collapse_subpackage_nodes(graph): if master_meta.name() == meta.name(): master = True group = node_groups.get(meta_path, {}) - subgroup = group.get(HashableDict(meta.config.variant), {}) + subgroup = group.get(deepfreeze(meta.config.variant), {}) if master: if 'master' in subgroup: raise ValueError("tried to set more than one node in a group as master") @@ -336,7 +354,7 @@ def collapse_subpackage_nodes(graph): sps = subgroup.get('subpackages', []) sps.append(node) subgroup['subpackages'] = sps - group[HashableDict(meta.config.variant)] = subgroup + group[deepfreeze(meta.config.variant)] = subgroup node_groups[meta_path] = group for recipe_path, group in node_groups.items(): @@ -409,11 +427,14 @@ def _fix_any(value, config): return value -@conda_interface.memoized +@lru_cache(maxsize=None) def _installable(name, version, build_string, config, conda_resolve): """Can Conda install the package we need?""" - ms = conda_interface.MatchSpec(" ".join([name, _fix_any(version, config), - _fix_any(build_string, config)])) + ms = MatchSpec( + " ".join( + [name, _fix_any(version, config), _fix_any(build_string, config)] + ) + ) installable = conda_resolve.find_matches(ms) if not installable: log.warn("Dependency {name}, version {ver} is not installable from your " @@ -435,7 +456,7 @@ def _buildable(name, version, recipes_dir, worker, config, finalize): path), worker, finalize=finalize)] # this is our target match - ms = conda_interface.MatchSpec(" ".join([name, _fix_any(version, config)])) + ms = MatchSpec(" ".join([name, _fix_any(version, config)])) available = False for m in metadata_tuples: available = match_peer_job(ms, m) diff --git a/.ci_support/linux64.yaml b/.ci_support/linux64.yaml index f6d599fd7c54e..f37e7a23966d3 100644 --- a/.ci_support/linux64.yaml +++ b/.ci_support/linux64.yaml @@ -1,3 +1,7 @@ +c_stdlib_version: +- 2.17 +c_stdlib: +- sysroot cdt_name: - cos7 c_compiler: @@ -15,7 +19,7 @@ target_platform: channel_sources: - conda-forge docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 cuda_compiler: - None cuda_compiler_version: diff --git a/.ci_support/linux64_cuda118.yaml b/.ci_support/linux64_cuda118.yaml index f2042b116862b..630d668061e97 100644 --- a/.ci_support/linux64_cuda118.yaml +++ b/.ci_support/linux64_cuda118.yaml @@ -14,6 +14,10 @@ cxx_compiler_version: - 11 fortran_compiler_version: - 11 +c_stdlib: +- sysroot +c_stdlib_version: +- 2.17 cdt_name: - cos7 target_platform: @@ -21,10 +25,10 @@ target_platform: channel_sources: - conda-forge docker_image: -- quay.io/condaforge/linux-anvil-cuda:11.8 +- quay.io/condaforge/linux-anvil-x86_64-cuda11.8:ubi8 cuda_compiler: - nvcc cuda_compiler_version: - 11.8 cuda_compiler_version_min: -- 11.2 +- 11.8 diff --git a/.ci_support/linux64_cuda112.yaml b/.ci_support/linux64_cuda120.yaml similarity index 71% rename from .ci_support/linux64_cuda112.yaml rename to .ci_support/linux64_cuda120.yaml index 4f8476993dcb2..3ee9ebaaf1915 100644 --- a/.ci_support/linux64_cuda112.yaml +++ b/.ci_support/linux64_cuda120.yaml @@ -9,11 +9,15 @@ go_compiler: cgo_compiler: - go-cgo c_compiler_version: -- 10 +- 12 cxx_compiler_version: -- 10 +- 12 fortran_compiler_version: -- 10 +- 12 +c_stdlib: +- sysroot +c_stdlib_version: +- 2.17 cdt_name: - cos7 target_platform: @@ -21,10 +25,10 @@ target_platform: channel_sources: - conda-forge docker_image: -- quay.io/condaforge/linux-anvil-cuda:11.2 +- quay.io/condaforge/linux-anvil-x86_64:alma9 cuda_compiler: -- nvcc +- cuda-nvcc cuda_compiler_version: -- 11.2 +- 12.0 cuda_compiler_version_min: -- 11.2 +- 11.8 diff --git a/.ci_support/linux_aarch64.yaml b/.ci_support/linux_aarch64.yaml new file mode 100644 index 0000000000000..ce52634c61904 --- /dev/null +++ b/.ci_support/linux_aarch64.yaml @@ -0,0 +1,28 @@ +c_stdlib_version: +- 2.17 +c_stdlib: +- sysroot +cdt_name: +- cos7 +c_compiler: +- gcc +cxx_compiler: +- gxx +fortran_compiler: +- gfortran +go_compiler: +- go-nocgo +cgo_compiler: +- go-cgo +target_platform: +- linux-aarch64 +channel_sources: +- conda-forge +docker_image: +- quay.io/condaforge/linux-anvil-aarch64:alma9 +cuda_compiler: +- None +cuda_compiler_version: +- None +cuda_compiler_version_min: +- None diff --git a/.ci_support/osx64.yaml b/.ci_support/osx64.yaml index a09f5d08dc5c3..e4efc34aa5221 100644 --- a/.ci_support/osx64.yaml +++ b/.ci_support/osx64.yaml @@ -2,6 +2,12 @@ c_compiler: - clang cxx_compiler: - clangxx +c_stdlib: + - macosx_deployment_target +c_stdlib_version: + - 10.13 +MACOSX_DEPLOYMENT_TARGET: + - 10.13 fortran_compiler: - gfortran go_compiler: diff --git a/.ci_support/osx_arm64.yaml b/.ci_support/osx_arm64.yaml new file mode 100644 index 0000000000000..250422bb9b25f --- /dev/null +++ b/.ci_support/osx_arm64.yaml @@ -0,0 +1,20 @@ +c_compiler: + - clang +cxx_compiler: + - clangxx +c_stdlib: + - macosx_deployment_target +c_stdlib_version: + - 11.0 +MACOSX_DEPLOYMENT_TARGET: + - 11.0 +fortran_compiler: + - gfortran +go_compiler: + - go-nocgo +cgo_compiler: + - go-cgo +channel_sources: + - conda-forge +target_platform: + - osx-arm64 diff --git a/.ci_support/requirements.txt b/.ci_support/requirements.txt deleted file mode 100644 index f8ade8f31d45c..0000000000000 --- a/.ci_support/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -conda>=23.7.3 -conda-libmamba-solver>=23.7.0 -conda-build>=3.16,!=3.28.3 -conda-index>=0.3.0 -conda-forge-ci-setup=3.* -conda-forge-pinning -networkx=2.4 diff --git a/.ci_support/win64.yaml b/.ci_support/win64.yaml index b63abaadb6e42..47801b5ba6db6 100644 --- a/.ci_support/win64.yaml +++ b/.ci_support/win64.yaml @@ -1,5 +1,7 @@ go_compiler: - go-nocgo +c_stdlib: + - vs cgo_compiler: - go-cgo channel_sources: diff --git a/.gitattributes b/.gitattributes index 5423d792ba24b..8833b12e52c6b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,5 @@ *.yaml text eol=lf *.sh text eol=lf *.bat text eol=crlf +# GitHub syntax highlighting +pixi.lock linguist-language=YAML linguist-generated=true diff --git a/.github/ISSUE_TEMPLATE/bugs_template.yml b/.github/ISSUE_TEMPLATE/bugs_template.yml index 4e1b54fe4acf1..5b63ff3a8bac7 100644 --- a/.github/ISSUE_TEMPLATE/bugs_template.yml +++ b/.github/ISSUE_TEMPLATE/bugs_template.yml @@ -12,7 +12,7 @@ body: 1. If the issue is related to the staged-recipes infrastructure, ping the `@conda-forge/staged-recipes` team in this issue. _Note:_ If you're not a member of the conda-forge GitHub organization, this will be disabled by GitHub and you can ask the bot to ping the team for you by entering the following command in a comment: `@conda-forge-admin, please ping conda-forge/staged-recipes` 2. If the issue is related to conda-forge, please open an issue in the [general conda-forge repo](https://github.com/conda-forge/conda-forge.github.io). - 3. If you need help, join our [gitter](https://gitter.im/conda-forge/conda-forge.github.io) community chat room. + 3. If you need help, join our [Zulip](https://conda-forge.zulipchat.com) community chat. - type: textarea id: comment diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 1da0098334df9..e2690f4ee69e4 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: true contact_links: - - name: Conda-forge gitter chat - url: https://gitter.im/conda-forge/conda-forge.github.io + - name: Conda-forge Zulip chat + url: https://conda-forge.zulipchat.com about: Chat to us about conda-forge and ask general questions. - name: Conda-forge documentation url: https://conda-forge.org/docs/ diff --git a/.github/ISSUE_TEMPLATE/issue_template.yml b/.github/ISSUE_TEMPLATE/issue_template.yml index 75d93d4b3337b..a992ad7162752 100644 --- a/.github/ISSUE_TEMPLATE/issue_template.yml +++ b/.github/ISSUE_TEMPLATE/issue_template.yml @@ -12,7 +12,7 @@ body: 1. If the issue is related to the staged-recipes infrastructure, ping the `@conda-forge/staged-recipes` team in this issue. _Note:_ If you're not a member of the conda-forge GitHub organization, this will be disabled by GitHub and you can ask the bot to ping the team for you by entering the following command in a comment: `@conda-forge-admin, please ping conda-forge/staged-recipes` 2. If the issue is related to conda-forge, please open an issue in the [general conda-forge repo](https://github.com/conda-forge/conda-forge.github.io). - 3. If you need help, join our [gitter](https://gitter.im/conda-forge/conda-forge.github.io) community chat room. + 3. If you need help, join our [Zulip](https://conda-forge.zulipchat.com) community chat. - type: textarea id: comment diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000000..5f454fdfb7985 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + github-actions: + patterns: + - '*' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5691d8ea0f393..325a8059a4fe1 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,6 +16,7 @@ right people know. There are language-specific teams for reviewing recipes. | Julia | `@conda-forge/help-julia` | | ruby | `@conda-forge/help-ruby` | | Rust | `@conda-forge/help-rust` | +| Go | `@conda-forge/help-go` | | other | `@conda-forge/staged-recipes` | Once the PR is ready for review, please mention one of the teams above in a @@ -25,11 +26,11 @@ Then, a bot will label the PR as 'review-requested'. Due to GitHub limitations, first time contributors to conda-forge are unable to ping conda-forge teams directly, but you can [ask a bot to ping the team][1] using a special command in a comment on the PR to get the attention of the -`staged-recipes` team. You can also consider asking on our [Gitter channel][2] +`staged-recipes` team. You can also consider asking on our [Zulip chat][2] if your recipe isn't reviewed promptly. [1]: https://conda-forge.org/docs/maintainer/infrastructure.html#conda-forge-admin-please-ping-team -[2]: https://gitter.im/conda-forge/conda-forge.github.io +[2]: https://conda-forge.zulipchat.com All apologies in advance if your recipe PR does not receive prompt attention. This is a high volume repository and the reviewers are volunteers. Review times diff --git a/.github/workflows/automate-review-labels.yml b/.github/workflows/automate-review-labels.yml index 8aceab5843d47..84c82cd4af072 100644 --- a/.github/workflows/automate-review-labels.yml +++ b/.github/workflows/automate-review-labels.yml @@ -1,139 +1,136 @@ - name: 'Automated review labels' +name: 'Automated review labels' - on: - issue_comment: - types: [created] - issues: - types: [unlabeled, labeled] - pull_request_target: - types: [unlabeled, labeled] +on: + issue_comment: + types: [ created ] + issues: + types: [ unlabeled, labeled ] + pull_request_target: + types: [ unlabeled, labeled ] - permissions: - issues: write # for adding label to an issue - pull-requests: write # for adding label to a pr +permissions: + issues: write # for adding label to an issue + pull-requests: write # for adding label to a pr - jobs: +jobs: - add-review-team-label: - name: 'When pinged, label a PR with review team' - if: > - github.event.issue - && github.event.issue.pull_request - runs-on: ubuntu-latest - steps: - - name: check-teams - id: check_teams - uses: actions/github-script@v6 - with: - script: | - const teams = [ - '@conda-forge/staged-recipes', - '@conda-forge/help-c-cpp', - '@conda-forge/help-cdts', - '@conda-forge/help-go', - '@conda-forge/help-java', - '@conda-forge/help-julia', - '@conda-forge/help-nodejs', - '@conda-forge/help-perl', - '@conda-forge/help-python', - '@conda-forge/help-python-c', - '@conda-forge/help-r', - '@conda-forge/help-ruby' - ]; - let found_label = false; - for (const team of teams) { - let text = context.payload.comment.body; - const regex = new RegExp(team + '[^\-]|' + team + '$'); - let result = regex.test(text); - if (result) { - const slug = team.replace("@conda-forge/", ""); - const label = slug.replace("help-", ""); - found_label = true; - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: [label, 'review-requested'] + add-review-team-label: + name: 'When pinged, label a PR with review team' + if: > + github.event.issue && github.event.issue.pull_request + runs-on: ubuntu-latest + steps: + - name: check-teams + id: check_teams + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const teams = [ + '@conda-forge/staged-recipes', + '@conda-forge/help-c-cpp', + '@conda-forge/help-cdts', + '@conda-forge/help-go', + '@conda-forge/help-java', + '@conda-forge/help-julia', + '@conda-forge/help-nodejs', + '@conda-forge/help-perl', + '@conda-forge/help-python', + '@conda-forge/help-python-c', + '@conda-forge/help-r', + '@conda-forge/help-ruby', + '@conda-forge/help-rust' + ]; + let found_label = false; + for (const team of teams) { + let text = context.payload.comment.body; + const regex = new RegExp(team + '[^\w-]|' + team + '$'); + let result = regex.test(text); + if (result) { + const slug = team.replace("@conda-forge/", ""); + const label = slug.replace("help-", ""); + found_label = true; + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: [label, 'review-requested'] + }); + // NOTE: GitHub Actions default token lacks permission to + // assign teams for review; external bot required for + // that feature. + // + https://github.com/conda-forge/staged-recipes/issues/18023#issuecomment-1080451231 + console.log(`Somebody mentioned ${slug}.`); + if (label == "staged-recipes") { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: 'To help direct your pull request to the best reviewers, ' + + 'please mention a topic-specifc team if your recipe matches any of the following: ' + + 'conda-forge/help-c-cpp, ' + + 'conda-forge/help-cdts, ' + + 'conda-forge/help-go, ' + + 'conda-forge/help-java, ' + + 'conda-forge/help-julia, ' + + 'conda-forge/help-nodejs, ' + + 'conda-forge/help-perl, ' + + 'conda-forge/help-python, ' + + 'conda-forge/help-python-c, ' + + 'conda-forge/help-r, ' + + 'conda-forge/help-ruby,' + + 'or ' + + 'conda-forge/help-rust' + + '. ' + + 'Thanks!' }); - // NOTE: GitHub Actions default token lacks permission to - // assign teams for review; external bot required for - // that feature. - // - https://github.com/conda-forge/staged-recipes/issues/18023#issuecomment-1080451231 - console.log(`Somebody mentioned ${slug}.`); - if (label == "staged-recipes") { - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'To help direct your pull request to the best reviewers, ' + - 'please mention a topic-specifc team if your recipe matches any of the following: ' + - 'conda-forge/help-c-cpp, ' + - 'conda-forge/help-cdts, ' + - 'conda-forge/help-go, ' + - 'conda-forge/help-java, ' + - 'conda-forge/help-julia, ' + - 'conda-forge/help-nodejs, ' + - 'conda-forge/help-perl, ' + - 'conda-forge/help-python, ' + - 'conda-forge/help-python-c, ' + - 'conda-forge/help-r, ' + - 'conda-forge/help-ruby,' + - 'or ' + - 'conda-forge/help-rust' + - '. ' + - 'Thanks!' - }); - } - } - } - return found_label; - - name: remove-labels - if: > - (steps.check_teams.outputs.result == 'true') - && contains(github.event.issue.labels.*.name, 'Awaiting author contribution') - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: ['Awaiting author contribution'] - }) + } + } + } + return found_label; + - name: remove-labels + if: > + (steps.check_teams.outputs.result == 'true') && contains(github.event.issue.labels.*.name, 'Awaiting author contribution') + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: ['Awaiting author contribution'] + }) - add-await-when-review-removed: - name: 'Add awaiting-author when review-requested removed' - if: > - github.event.action == 'unlabeled' - && github.event.label.name == 'review-requested' - runs-on: ubuntu-latest - steps: - - name: add-labels - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['Awaiting author contribution'] - }); + add-await-when-review-removed: + name: 'Add awaiting-author when review-requested removed' + if: > + github.event.action == 'unlabeled' && github.event.label.name == 'review-requested' + runs-on: ubuntu-latest + steps: + - name: add-labels + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['Awaiting author contribution'] + }); - remove-review-when-await-added: - name: 'Removed review-requested when awaiting-author added' - if: > - github.event.action == 'labeled' - && github.event.label.name == 'Awaiting author contribution' - runs-on: ubuntu-latest - steps: - - name: remove-labels - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: ['review-requested'] - }) + remove-review-when-await-added: + name: 'Removed review-requested when awaiting-author added' + if: > + github.event.action == 'labeled' && github.event.label.name == 'Awaiting author contribution' + runs-on: ubuntu-latest + steps: + - name: remove-labels + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: ['review-requested'] + }) diff --git a/.github/workflows/correct_directory.yml b/.github/workflows/correct_directory.yml deleted file mode 100644 index c471322711888..0000000000000 --- a/.github/workflows/correct_directory.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: directory_linter - -on: - pull_request_target: - paths: - - 'recipes/*.yml' - - 'recipes/*.yaml' - -jobs: - comment: - name: Notify user about wrong dir - runs-on: "ubuntu-latest" - steps: - - uses: actions/checkout@v2 - - - name: Comment on PR - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'Hi! Thanks for your contribution to conda-forge.' + - '\nUnfortunately, the recipe was added directly in the `recipes` folder without its own subfolder.\n' + - 'Please move the recipe file into a folder with the name of the package you want to submit.\n\n' + - 'For example: if your recipe is currently under `recipes/.yaml`, ' + - 'it should be moved to `recipes//meta.yaml`.\n' + - 'Thanks!' - }) diff --git a/.github/workflows/create_feedstocks.yml b/.github/workflows/create_feedstocks.yml index 6c2e9dab52359..2b4686d8ee862 100644 --- a/.github/workflows/create_feedstocks.yml +++ b/.github/workflows/create_feedstocks.yml @@ -5,7 +5,7 @@ on: branches: - main schedule: - - cron: '*/10 * * * *' + - cron: '*/10 * * * *' workflow_dispatch: null permissions: {} @@ -21,19 +21,19 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: token: ${{ github.token }} - name: Prevent multiple jobs running in parallel id: conversion_lock - uses: beckermr/turnstyle-python@v1 + uses: beckermr/turnstyle-python@8f1ceb87dabbbbebe42257b85c368f6110bb9170 # v2 with: abort-after-seconds: 3 poll-interval-seconds: 2 github-token: ${{ secrets.GITHUB_TOKEN }} continue-on-error: true - + - name: commit any changes upon checkout run: | git config --global user.email "pelson.pub+conda-forge@gmail.com" @@ -49,15 +49,15 @@ jobs: # outcome is evaluated before continue-on-error above if: ${{ steps.conversion_lock.outcome == 'success' }} run: | - # Avoid wasting CI time if there are no recipes ready for conversion - if [ "$(ls recipes/*/meta.yaml | grep -v recipes/example/meta.yaml --count)" -eq 0 ]; then - echo "No new recipes found, exiting..." - exit 0 - fi + # Avoid wasting CI time if there are no recipes ready for conversion + if [ "$(ls recipes/*/meta.yaml | grep -v recipes/example/meta.yaml --count)" -eq 0 ]; then + echo "No new recipes found, exiting..." + exit 0 + fi - echo "Creating feedstocks from the recipe(s)." + echo "Creating feedstocks from the recipe(s)." - source ./.github/workflows/scripts/create_feedstocks + source ./.github/workflows/scripts/create_feedstocks env: STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} GH_TOKEN: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }} diff --git a/.github/workflows/do_not_edit_example.yml b/.github/workflows/do_not_edit_example.yml deleted file mode 100644 index dce30d1768695..0000000000000 --- a/.github/workflows/do_not_edit_example.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: do_not_edit_example - -on: - pull_request_target: - paths: - - 'recipes/example/meta.yaml' - -jobs: - comment: - name: Notify user about not editing example recipe - runs-on: "ubuntu-latest" - steps: - - uses: actions/checkout@v2 - - - name: Comment on PR - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'Hi! Thanks for your contribution to conda-forge.' + - '\nWhen submitting a pull request, please do not change anything in the example recipe.\n' + - 'Please make sure that any changes are reverted before you submit it for review.\n' + - 'Thanks!' - }) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000000000..30237be21f441 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,54 @@ +name: staged-recipes linter + +on: + # this workflow requires access to a read-only github token + # and potentially runs untrusted code via python's `eval` + # thus we can only run it on pull_request events so that the + # the token does not have any write permissions + pull_request: + types: + - opened + - synchronize + - reopened + - labeled + - unlabeled + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + linter: + name: linter + runs-on: ubuntu-latest + defaults: + run: + shell: bash -leo pipefail {0} + + steps: + - name: checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: install code + uses: mamba-org/setup-micromamba@06375d89d211a1232ef63355742e9e2e564bc7f7 + with: + environment-name: lint + condarc: | + show_channel_urls: true + channel_priority: strict + channels: + - conda-forge + create-args: >- + conda-smithy + pygithub + requests + pixi + + - name: lint + run: | + python .github/workflows/scripts/linter.py \ + --owner=${{ github.repository_owner }} \ + --pr-num=${{ github.event.number }} \ + >> $GITHUB_STEP_SUMMARY + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/linter_make_comment.yml b/.github/workflows/linter_make_comment.yml new file mode 100644 index 0000000000000..8ae7a6753261a --- /dev/null +++ b/.github/workflows/linter_make_comment.yml @@ -0,0 +1,42 @@ +name: make comment for staged-recipes linter + +on: + workflow_run: + workflows: ["staged-recipes linter"] + types: + - completed + +jobs: + make-comment: + name: make comment + runs-on: ubuntu-latest + defaults: + run: + shell: bash -leo pipefail {0} + + steps: + - name: checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: install code + uses: mamba-org/setup-micromamba@06375d89d211a1232ef63355742e9e2e564bc7f7 + with: + environment-name: lint + condarc: | + show_channel_urls: true + channel_priority: strict + channels: + - conda-forge + create-args: >- + conda-smithy + pygithub + requests + + - name: make comment + run: | + python .github/workflows/scripts/linter_make_comment.py \ + --head-repo-owner=${{ github.event.workflow_run.head_repository.owner.login }} \ + --workflow-run-id=${{ github.event.workflow_run.id }} \ + --head-sha=${{ github.event.workflow_run.head_sha }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scripts/create_feedstocks b/.github/workflows/scripts/create_feedstocks index e28bba26925cb..801edb44a61ae 100755 --- a/.github/workflows/scripts/create_feedstocks +++ b/.github/workflows/scripts/create_feedstocks @@ -38,14 +38,17 @@ touch ~/Miniforge3/conda-meta/pinned source ~/Miniforge3/bin/activate conda install --yes --quiet \ - conda-forge-ci-setup=3.* \ - "conda-smithy>=3.7.1,<4.0.0a0" \ + conda-forge-ci-setup=4.* \ + "conda-smithy>=3.38.0,<4.0.0a0" \ conda-forge-pinning \ - "conda-build>=3.16,!=3.28.3" \ + "conda-build>=24.3" \ "gitpython>=3.0.8,<3.1.20" \ requests \ ruamel.yaml \ - "pygithub>=2.1.1" + "pygithub>=2.1.1" \ + "rattler-build-conda-compat>=1.2.0,<2.0.0a0" \ + "conda-forge-feedstock-ops>=0.5.0" \ + "conda-forge-metadata>=0.8.1" conda info mamba info diff --git a/.github/workflows/scripts/create_feedstocks.py b/.github/workflows/scripts/create_feedstocks.py index 47608c4e212bb..c99b22b43d0f9 100755 --- a/.github/workflows/scripts/create_feedstocks.py +++ b/.github/workflows/scripts/create_feedstocks.py @@ -10,9 +10,15 @@ export GH_TOKEN=$(cat ~/.conda-smithy/github.token) """ + from __future__ import print_function +from __future__ import annotations +import json +from pathlib import Path +from typing import Iterator from conda_build.metadata import MetaData +from rattler_build_conda_compat.render import MetaData as RattlerBuildMetaData from conda_smithy.utils import get_feedstock_name_from_meta from contextlib import contextmanager from datetime import datetime, timezone @@ -25,32 +31,92 @@ import traceback import time +import github import requests from ruamel.yaml import YAML +from conda_forge_feedstock_ops.parse_package_and_feedstock_names import ( + parse_package_and_feedstock_names +) +from conda_forge_metadata.feedstock_outputs import sharded_path as _get_sharded_path # Enable DEBUG to run the diagnostics, without actually creating new feedstocks. DEBUG = False REPO_SKIP_LIST = ["core", "bot", "staged-recipes", "arm-arch", "systems", "ctx"] -recipe_directory_name = 'recipes' +recipe_directory_name = "recipes" + +def _test_and_raise_besides_file_not_exists(e: github.GithubException): + if isinstance(e, github.UnknownObjectException): + return + if e.status == 404 and "No object found" in e.data["message"]: + return + raise e -def list_recipes(): - if os.path.isdir(recipe_directory_name): - recipes = os.listdir(recipe_directory_name) + +def _register_package_for_feedstock(feedstock, pkg_name, gh): + repo = gh.get_repo("conda-forge/feedstock-outputs") + try: + contents = repo.get_contents(_get_sharded_path(pkg_name)) + except github.GithubException as e: + _test_and_raise_besides_file_not_exists(e) + contents = None + + if contents is None: + data = {"feedstocks": [feedstock]} + repo.create_file( + _get_sharded_path(pkg_name), + f"[cf admin skip] ***NO_CI*** add output {pkg_name} for conda-forge/{feedstock}-feedstock", + json.dumps(data), + ) + print(f" output {pkg_name} added for feedstock {feedstock}", flush=True) else: - recipes = [] + # we proceed anyways and do not raise since it could be a rerun of staged recipes + # print a warning for the users + data = json.loads(contents.decoded_content.decode("utf-8")) + print(f" WARNING: output {pkg_name} already exists from feedstock(s) {data["feedstocks"]}", flush=True) + + +def list_recipes() -> Iterator[tuple[str, str]]: + """ + Locates all the recipes in the `recipes/` folder at the root of the repository. + + For each found recipe this function returns a tuple consisting of + * the path to the recipe directory + * the name of the feedstock + """ + repository_root = Path(__file__).parent.parent.parent.parent.absolute() + repository_recipe_dir = repository_root / recipe_directory_name - for recipe_dir in recipes: + # Ignore if the recipe directory does not exist. + if not repository_recipe_dir.is_dir: + return + + for recipe_dir in repository_recipe_dir.iterdir(): # We don't list the "example" feedstock. It is an example, and is there # to be helpful. # .DS_Store is created by macOS to store custom attributes of its # containing folder. - if recipe_dir in ['example', '.DS_Store']: + if recipe_dir.name in ["example", "example-v1", ".DS_Store"]: + continue + + # Try to look for a conda-build recipe. + absolute_feedstock_path = repository_recipe_dir / recipe_dir + try: + yield ( + str(absolute_feedstock_path), + get_feedstock_name_from_meta(MetaData(absolute_feedstock_path)), + ) continue - path = os.path.abspath(os.path.join(recipe_directory_name, recipe_dir)) - yield path, get_feedstock_name_from_meta(MetaData(path)) + except OSError: + pass + + # If no conda-build recipe was found, try to load a rattler-build recipe. + yield ( + str(absolute_feedstock_path), + get_feedstock_name_from_meta(RattlerBuildMetaData(absolute_feedstock_path)), + ) @contextmanager @@ -206,7 +272,7 @@ def write_token(name, token): # gh_travis = Github(os.environ['GH_TRAVIS_TOKEN']) gh_travis = None - + gh = None if 'GH_TOKEN' in os.environ: write_token('github', os.environ['GH_TOKEN']) @@ -370,9 +436,11 @@ def write_token(name, token): if not feedstock_token_exists("conda-forge", name + "-feedstock"): subprocess.check_call( ['conda', 'smithy', 'generate-feedstock-token', + '--unique-token-per-provider', '--feedstock_directory', feedstock_dir] + owner_info) subprocess.check_call( ['conda', 'smithy', 'register-feedstock-token', + '--unique-token-per-provider', '--without-circle', '--without-drone', '--feedstock_directory', feedstock_dir] + owner_info) @@ -398,6 +466,12 @@ def write_token(name, token): ) subprocess.check_call( ['conda', 'smithy', 'rerender', '--no-check-uptodate'], cwd=feedstock_dir) + + # pre-register outputs + print("registering outputs...") + _, pkg_names, _ = parse_package_and_feedstock_names(feedstock_dir, use_container=False) + for pkg_name in pkg_names: + _register_package_for_feedstock(name, pkg_name, gh) except subprocess.CalledProcessError: exit_code = 0 traceback.print_exception(*sys.exc_info()) diff --git a/.github/workflows/scripts/linter.py b/.github/workflows/scripts/linter.py new file mode 100644 index 0000000000000..05adff4b459f3 --- /dev/null +++ b/.github/workflows/scripts/linter.py @@ -0,0 +1,308 @@ +import argparse +import os +import sys +from collections import defaultdict +from pathlib import Path +from subprocess import check_output + +from conda_smithy.linter import conda_recipe_v1_linter +from conda_smithy.linter.utils import ( + get_section, +) +from conda_smithy.utils import get_yaml, render_meta_yaml +import github +import requests + +NOCOMMENT_REQ_TEAMS = ["conda-forge/r", "conda-forge/cuda"] +HERE = Path(__file__).parent +ROOT = (HERE / ".." / ".." / "..").absolute() + + +def _test_and_raise_besides_file_not_exists(e: github.GithubException): + if isinstance(e, github.UnknownObjectException): + return + if e.status == 404 and "No object found" in e.data["message"]: + return + raise e + + +def _lint_recipes(gh, pr): + lints = defaultdict(list) + hints = defaultdict(list) + fnames = set(f.filename for f in pr.get_files()) + labels = set(label.name for label in pr.get_labels()) + extra_edits = False + example_recipes = ["recipes/example/meta.yaml", "recipes/example-v1/recipe.yaml"] + + # 1. Do not edit or delete example recipes and only edit recipe files + if "maintenance" not in labels: + for fname in fnames: + if fname in example_recipes: + lints[fname].append("Do not edit or delete example recipes in `recipes/example/` or `recipe/example-v1/`.") + extra_edits = True + if not fname.startswith("recipes/"): + lints[fname].append("Do not edit files outside of the `recipes/` directory.") + extra_edits = True + + # 2. Make sure the new recipe is in the right directory + for fname in fnames: + if ( + fname.startswith("recipes/example/") + or fname.startswith("recipes/example-v1/") + or fname.startswith("recipes/meta.y") + or fname.startswith("recipes/recipe.y") + ) and fname not in example_recipes: + lints[fname].append( + "Please put your recipe in its own directory in the `recipes/` directory as " + "`recipe//.yaml`." + ) + + # 3. Ensure environment.yaml and pixi.toml are in sync + original_environment_yaml = (ROOT / "environment.yaml").read_text() + pixi_exported_env_yaml = check_output( + ["pixi", "project", "export", "conda-environment", "-e", "build"], + text=True, + ) + if original_environment_yaml != pixi_exported_env_yaml: + import difflib + + _orig_lines = original_environment_yaml.splitlines(keepends=True) + _expt_lines = pixi_exported_env_yaml.splitlines(keepends=True) + print("environment diff:", flush=True) + print(''.join(difflib.unified_diff(_orig_lines, _expt_lines)), flush=True) + lints["environment.yaml"].append( + "The `environment.yaml` file is out of sync with `pixi.toml`. " + "Fix by running `pixi project export conda-environment -e build > environment.yaml`." + ) + + # Recipe-specific lints/hints + for fname in fnames: + if ( + (not fname.startswith("recipes/")) + or (not fname.endswith(".yaml")) + or fname in example_recipes + ): + continue + + # grab basic metadata + if fname.endswith("meta.yaml"): + with open(fname) as fh: + content = render_meta_yaml("".join(fh)) + meta = get_yaml().load(content) + recipe_version = 0 + else: + meta = get_yaml().load(Path(fname)) + recipe_version = 1 + + package_section = get_section( + meta, "package", lints, recipe_version=recipe_version + ) + sources_section = get_section( + meta, "source", lints, recipe_version=recipe_version + ) + outputs_section = get_section( + meta, "outputs", lints, recipe_version=recipe_version + ) + extra_section = get_section( + meta, "extra", lints, recipe_version=recipe_version + ) + maintainers = extra_section.get("recipe-maintainers", []) + + if recipe_version == 1: + recipe_name = conda_recipe_v1_linter.get_recipe_name(meta) + else: + recipe_name = package_section.get("name", "").strip() + + recipe_name = extra_section.get("feedstock-name", recipe_name) + + # 4. Check for existing feedstocks in conda-forge or bioconda + if recipe_name: + cf = gh.get_user("conda-forge") + + for name in set( + [ + recipe_name, + recipe_name.replace("-", "_"), + recipe_name.replace("_", "-"), + ] + ): + try: + if cf.get_repo(f"{name}-feedstock"): + existing_recipe_name = name + feedstock_exists = True + break + else: + feedstock_exists = False + except github.GithubException as e: + _test_and_raise_besides_file_not_exists(e) + feedstock_exists = False + + if feedstock_exists and existing_recipe_name == recipe_name: + lints[fname].append("Feedstock with the same name exists in conda-forge.") + elif feedstock_exists: + hints[fname].append( + f"Feedstock with the name {existing_recipe_name} exists in conda-forge. " + f"Is it the same as this package ({recipe_name})?" + ) + + bio = gh.get_user("bioconda").get_repo("bioconda-recipes") + try: + bio.get_dir_contents(f"recipes/{recipe_name}") + except github.GithubException as e: + _test_and_raise_besides_file_not_exists(e) + else: + hints[fname].append( + "Recipe with the same name exists in bioconda: " + "please discuss with @conda-forge/bioconda-recipes." + ) + + url = None + if recipe_version == 1: + for source_section in sources_section: + if str(source_section.get("url")).startswith("https://pypi.io/packages/source/"): + url = source_section["url"] + else: + for source_section in sources_section: + if str(source_section.get("url")).startswith( + "https://pypi.io/packages/source/" + ): + url = source_section["url"] + if url: + # get pypi name from urls like "https://pypi.io/packages/source/b/build/build-0.4.0.tar.gz" + pypi_name = url.split("/")[6] + mapping_request = requests.get( + "https://raw.githubusercontent.com/regro/cf-graph-countyfair/master/mappings/pypi/name_mapping.yaml" + ) + if mapping_request.status_code == 200: + mapping_raw_yaml = mapping_request.content + mapping = get_yaml().load(mapping_raw_yaml) + for pkg in mapping: + if pkg.get("pypi_name", "") == pypi_name: + conda_name = pkg["conda_name"] + hints[fname].append( + f"A conda package with same name ({conda_name}) already exists." + ) + + # 5. Ensure all maintainers have commented that they approve of being listed + if maintainers: + # Get PR author, issue comments, and review comments + pr_author = pr.user.login + issue_comments = pr.get_issue_comments() + review_comments = pr.get_reviews() + + # Combine commenters from both issue comments and review comments + commenters = {comment.user.login for comment in issue_comments} + commenters.update({review.user.login for review in review_comments}) + + # Check if all maintainers have either commented or are the PR author + non_participating_maintainers = set() + for maintainer in maintainers: + if ( + maintainer not in commenters + and maintainer != pr_author + and maintainer not in NOCOMMENT_REQ_TEAMS + ): + non_participating_maintainers.add(maintainer) + + # Add a lint message if there are any non-participating maintainers + if non_participating_maintainers: + lints[fname].append( + f"The following maintainers have not yet confirmed that they are willing to be listed here: " + f"{', '.join(non_participating_maintainers)}. Please ask them to comment on this PR if they are." + ) + + # 6. Only conda-forge teams can be maintainers + if maintainers: + for maintainer in maintainers: + if "/" in maintainer: + res = maintainer.split("/", 1) + if len(res) == 2: + org, team = res + if org != "conda-forge": + lints[fname].append( + "Only conda-forge teams can be team maintainers." + f" {maintainer} is not a team in conda-forge." + ) + else: + lints[fname].append( + f'Maintainer team "{maintainer}" is not in the ' + 'correct format. Please use "org/team" format.' + ) + + # 7. Check that feedstock-name is defined if recipe is multi-output + if outputs_section and ("feedstock-name" not in extra_section): + hints[fname].append( + "It looks like you are submitting a multi-output recipe. " + "In these cases, the correct name for the feedstock is ambiguous, " + "and our infrastructure defaults to the top-level `package.name` field. " + "Please add a `feedstock-name` entry in the `extra` section." + ) + + return dict(lints), dict(hints), extra_edits + + +def _comment_on_pr(pr, lints, hints, extra_edits): + if lints: + topline = "I found some lint." + elif hints: + topline = "your PR looks excellent but I have some suggestions." + else: + topline = "your PR looks excellent! :rocket:" + summary = f"Hi! This is the staged-recipes linter and {topline}\n" + + if extra_edits: + summary += """\ +\nIt looks like some changes were made outside the `recipes/` directory. \ +To ensure everything runs smoothly, please make sure that recipes are only \ +added to the `recipes/` directory and no other files are changed. + +If these changes are intentional (and you aren't submitting a recipe), \ +please add a `maintenance` label to the PR.\n""" + + added_lint_hint_header = False + all_fnames = set(lints.keys()) | set(hints.keys()) + for fname in all_fnames: + lint_message = "" + hint_message = "" + + if fname in lints and lints[fname]: + lint_message = " - lints:\n" + for lint in lints[fname]: + if lint: + lint_message += f" - {lint}\n" + + if fname in hints and hints[fname]: + hint_message = " - hints:\n" + for hint in hints[fname]: + if hint: + hint_message += f" - {hint}\n" + + if lint_message or hint_message: + if not added_lint_hint_header: + summary += "\nFile-specific lints and/or hints:\n" + added_lint_hint_header = True + summary += f"\n- `{fname}`:\n" + summary += lint_message + hint_message + "\n" + + print(summary) + print("###START-OF-SUMMARY###", file=sys.stderr) + print(summary, file=sys.stderr) + print("###END-OF-SUMMARY###", file=sys.stderr) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Lint staged recipes.') + parser.add_argument('--owner', type=str, required=True, help='the repo owner') + parser.add_argument('--pr-num', type=int, required=True, help='the PR number') + + args = parser.parse_args() + + gh = github.Github(auth=github.Auth.Token(os.getenv("GH_TOKEN"))) + repo = gh.get_repo(f"{args.owner}/staged-recipes") + pr = repo.get_pull(args.pr_num) + + lints, hints, extra_edits = _lint_recipes(gh, pr) + _comment_on_pr(pr, lints, hints, extra_edits) + if lints: + sys.exit(1) + diff --git a/.github/workflows/scripts/linter_make_comment.py b/.github/workflows/scripts/linter_make_comment.py new file mode 100644 index 0000000000000..ba7ea6175a606 --- /dev/null +++ b/.github/workflows/scripts/linter_make_comment.py @@ -0,0 +1,93 @@ +import argparse +import os + +import github +import requests + + +def _get_latest_run_summary(repo, workflow_run_id): + latest_run = repo.get_workflow_run(workflow_run_id) + for job in latest_run.jobs(): + pass + + r = requests.get(job.logs_url()) + + summary = "" + in_summary = False + for line in r.text.splitlines(): + line = line.strip() + if "###START-OF-SUMMARY###" in line: + in_summary = True + head_len = len(line.split("###START-OF-SUMMARY###")[0]) + continue + + if "###END-OF-SUMMARY###" in line: + in_summary = False + break + + if in_summary: + line = line[head_len:] + summary += line + "\n" + + return summary + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Lint staged recipes.') + parser.add_argument('--head-repo-owner', type=str, required=True, help='the head repo owner') + parser.add_argument('--workflow-run-id', type=int, required=True, help='the ID of the workflor run') + parser.add_argument('--head-sha', type=str, required=True, help='the head SHA of the PR') + + args = parser.parse_args() + + gh = github.Github(auth=github.Auth.Token(os.environ["GH_TOKEN"])) + head_repo = gh.get_repo(f"{args.head_repo_owner}/staged-recipes") + base_repo = gh.get_repo("conda-forge/staged-recipes") + + summary = _get_latest_run_summary(base_repo, args.workflow_run_id) + if summary: + print(summary) + commit = head_repo.get_commit(args.head_sha) + pr = None + for _pr in commit.get_pulls(): + if _pr.base.repo.full_name == base_repo.full_name: + pr = _pr + break + + if pr is None: + # for reasons I do not follow, sometimes the head commit API + # to get pull requests does not return the PR. + # So we try looking at PRs from base repo and find the same + # sha+head repo but limit the search since staged-recipes + # gets tons of PRs. + max_tries = 50 + num_tries = 0 + for _pr in base_repo.get_pulls(): + if ( + _pr.head.sha == args.head_sha + and _pr.head.repo.full_name == head_repo.full_name + ): + pr = _pr + break + num_tries += 1 + if num_tries == max_tries: + break + + if pr is not None: + comment = None + for _comment in pr.get_issue_comments(): + if "Hi! This is the staged-recipes linter" in _comment.body: + comment = _comment + + if comment: + if comment.body != summary: + curr_fline = comment.body.splitlines()[0].strip() + new_fl = summary.splitlines()[0].strip() + if curr_fline == new_fl: + comment.edit(summary) + else: + pr.create_issue_comment(summary) + else: + pr.create_issue_comment(summary) + else: + print("No PR found for the given commit. No comment being made!") diff --git a/.github/workflows/webservices.yml b/.github/workflows/webservices.yml new file mode 100644 index 0000000000000..d6f06b5c9c254 --- /dev/null +++ b/.github/workflows/webservices.yml @@ -0,0 +1,13 @@ +on: repository_dispatch + +jobs: + webservices: + runs-on: ubuntu-latest + name: webservices + steps: + - name: webservices + id: webservices + uses: conda-forge/webservices-dispatch-action@main + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + rerendering_github_token: ${{ secrets.RERENDERING_GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 88627c96ee05c..da9b4f6885679 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,18 @@ # macOS folder metadata .DS_Store +# macOS specific files +MacOSX*.sdk.tar.xz +SDKs/ + # User builds build_artifacts +miniforge3/ # Compiled Python code __pycache__ *.pyc +*.egg-info # Editor files *.swp @@ -16,3 +22,10 @@ __pycache__ # Jupyter checkpoints **/.ipynb_checkpoints/* + +# pixi environments +.pixi +pixi.lock + +# rattler-build outputs files +output/ diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 60c56d180aa1f..b0a1f5958ceb5 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -36,8 +36,16 @@ show_channel_urls: true solver: libmamba CONDARC +# Workaround for errors related to "unsafe" directories: +# https://github.blog/2022-04-12-git-security-vulnerability-announced/#cve-2022-24765 +git config --global --add safe.directory "${FEEDSTOCK_ROOT}" + # Copy the host recipes folder so we don't ever muck with it -cp -r ${FEEDSTOCK_ROOT} ~/staged-recipes-copy +# Skip build_artifacts and other big items because it gets huge with time +mkdir -p ~/staged-recipes-copy +shopt -s extglob dotglob +cp -r "${FEEDSTOCK_ROOT}"/!(.|..|build_artifacts|.pixi|miniforge3|MacOSX*.sdk.tar.xz|SDKs|output) ~/staged-recipes-copy +shopt -u extglob dotglob # Remove any macOS system files find ~/staged-recipes-copy/recipes -maxdepth 1 -name ".DS_Store" -delete @@ -47,15 +55,20 @@ echo "Pending recipes." ls -la ~/staged-recipes-copy/recipes echo "Finding recipes merged in main and removing them from the build." pushd "${FEEDSTOCK_ROOT}/recipes" > /dev/null -if [ "${AZURE}" == "True" ]; then +if [ "${CI:-}" != "" ]; then git fetch --force origin main:main fi -git ls-tree --name-only main -- . | xargs -I {} sh -c "rm -rf ~/staged-recipes-copy/recipes/{} && echo Removing recipe: {}" +shopt -s extglob dotglob +git ls-tree --name-only main -- !(example|example-v1) | xargs -I {} sh -c "rm -rf ~/staged-recipes-copy/recipes/{} && echo Removing recipe: {}" +shopt -u extglob dotglob popd > /dev/null - - -conda install --quiet --file ${FEEDSTOCK_ROOT}/.ci_support/requirements.txt +# Update environment +mv /opt/conda/conda-meta/history /opt/conda/conda-meta/history.$(date +%Y-%m-%d-%H-%M-%S) +echo > /opt/conda/conda-meta/history +micromamba install --root-prefix ~/.conda --prefix /opt/conda \ + --yes --override-channels --channel conda-forge --strict-channel-priority \ + --file "${FEEDSTOCK_ROOT}/environment.yaml" setup_conda_rc "${FEEDSTOCK_ROOT}" "/home/conda/staged-recipes-copy/recipes" "${CI_SUPPORT}/${CONFIG}.yaml" source run_conda_forge_build_setup @@ -66,12 +79,17 @@ find ~/staged-recipes-copy/recipes -mindepth 2 -maxdepth 2 -type f -name "yum_re xargs -r /usr/bin/sudo -n yum install -y # Make sure build_artifacts is a valid channel -conda index ${FEEDSTOCK_ROOT}/build_artifacts +conda index "${FEEDSTOCK_ROOT}/build_artifacts" ( endgroup "Configuring conda" ) 2> /dev/null echo "Building all recipes" -python ${CI_SUPPORT}/build_all.py +python "${CI_SUPPORT}/build_all.py" + +( startgroup "Inspecting artifacts" ) 2> /dev/null +# inspect_artifacts was only added in conda-forge-ci-setup 4.6.0; --all-packages in 4.9.3 +command -v inspect_artifacts >/dev/null 2>&1 && inspect_artifacts --all-packages || echo "inspect_artifacts needs conda-forge-ci-setup >=4.9.3" +( endgroup "Inspecting artifacts" ) 2> /dev/null ( startgroup "Final checks" ) 2> /dev/null diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 57a568798e1c9..08a935f03ba03 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -44,7 +44,7 @@ mkdir -p "$ARTIFACTS" DONE_CANARY="$ARTIFACTS/conda-forge-build-done" rm -f "$DONE_CANARY" -DOCKER_RUN_ARGS="-it" +DOCKER_RUN_ARGS="-it ${CONDA_FORGE_DOCKER_RUN_ARGS}" if [ "${AZURE}" == "True" ]; then DOCKER_RUN_ARGS="" diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index dfc54e56ef457..35ba13ed3dde2 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -4,21 +4,38 @@ set -x source .scripts/logging_utils.sh -( startgroup "Ensuring Miniforge" ) 2> /dev/null +( startgroup "Provisioning build tools" ) 2> /dev/null -MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" -MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh" -MINIFORGE_ROOT="${MINIFORGE_ROOT:-${HOME}/Miniforge3}" +MINIFORGE_HOME=${MINIFORGE_HOME:-${HOME}/miniforge3} +MINIFORGE_HOME=${MINIFORGE_HOME%/} # remove trailing slash +export CONDA_BLD_PATH=${CONDA_BLD_PATH:-${MINIFORGE_HOME}/conda-bld} -if [[ -d "${MINIFORGE_ROOT}" ]]; then - echo "Miniforge already installed at ${MINIFORGE_ROOT}." +if [[ -f "${MINIFORGE_HOME}/conda-meta/history" ]]; then + echo "Build tools already installed at ${MINIFORGE_HOME}." else - echo "Installing Miniforge" - curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" - bash $MINIFORGE_FILE -bp "${MINIFORGE_ROOT}" + if command -v micromamba >/dev/null 2>&1; then + micromamba_exe="micromamba" + echo "Found micromamba in PATH" + else + MICROMAMBA_VERSION="1.5.10-0" + if [[ "$(uname -m)" == "arm64" ]]; then + osx_arch="osx-arm64" + else + osx_arch="osx-64" + fi + MICROMAMBA_URL="https://github.com/mamba-org/micromamba-releases/releases/download/${MICROMAMBA_VERSION}/micromamba-${osx_arch}" + echo "Downloading micromamba ${MICROMAMBA_VERSION}" + micromamba_exe="$(mktemp -d)/micromamba" + curl -L -o "${micromamba_exe}" "${MICROMAMBA_URL}" + chmod +x "${micromamba_exe}" + fi + echo "Creating environment" + "${micromamba_exe}" create --yes --root-prefix ~/.conda --prefix "${MINIFORGE_HOME}" \ + --channel conda-forge \ + --file environment.yaml fi -( endgroup "Ensuring Miniforge" ) 2> /dev/null +( endgroup "Provisioning build tools" ) 2> /dev/null ( startgroup "Configuring conda" ) 2> /dev/null @@ -28,12 +45,9 @@ show_channel_urls: true solver: libmamba CONDARC -source "${MINIFORGE_ROOT}/etc/profile.d/conda.sh" +source "${MINIFORGE_HOME}/etc/profile.d/conda.sh" conda activate base -echo -e "\n\nInstalling conda-forge-ci-setup=3, conda-build." -conda install --quiet --file .ci_support/requirements.txt - echo -e "\n\nSetting up the condarc and mangling the compiler." setup_conda_rc ./ ./recipes ./.ci_support/${CONFIG}.yaml if [[ "${CI:-}" != "" ]]; then @@ -54,19 +68,42 @@ source run_conda_forge_build_setup set -e # make sure there is a package directory so that artifact publishing works -mkdir -p "${MINIFORGE_ROOT}/conda-bld/osx-64/" +mkdir -p "${CONDA_BLD_PATH}/osx-64/" "${CONDA_BLD_PATH}/osx-arm64/" "${CONDA_BLD_PATH}/noarch/" +# Make sure CONDA_BLD_PATH is a valid channel; only do it if noarch/repodata.json doesn't exist +# to save some time running locally +test -f "${CONDA_BLD_PATH}/noarch/repodata.json" || conda index "${CONDA_BLD_PATH}" -# Find the recipes from main in this PR and remove them. +# Find the recipes from upstream:main in this PR and remove them. echo "" echo "Finding recipes merged in main and removing them from the build." pushd ./recipes > /dev/null -git fetch --force origin main:main -git ls-tree --name-only main -- . | xargs -I {} sh -c "rm -rf {} && echo Removing recipe: {}" +if [ "${CI:-}" != "" ]; then + git fetch --force origin main:main +fi +shopt -s extglob dotglob +git ls-tree --name-only main -- !(example|example-v1) | xargs -I {} sh -c "rm -rf {} && echo Removing recipe: {}" +shopt -u extglob dotglob popd > /dev/null echo "" ( endgroup "Configuring conda" ) 2> /dev/null +# Set the target arch or auto detect it +if [[ -z "${TARGET_ARCH}" ]]; then + if [[ "$(uname -m)" == "arm64" ]]; then + TARGET_ARCH="arm64" + else + TARGET_ARCH="64" + fi +else + echo "TARGET_ARCH is set to ${TARGET_ARCH}" +fi + # We just want to build all of the recipes. echo "Building all recipes" -python .ci_support/build_all.py +python .ci_support/build_all.py --arch ${TARGET_ARCH} + +( startgroup "Inspecting artifacts" ) 2> /dev/null +# inspect_artifacts was only added in conda-forge-ci-setup 4.6.0; --all-packages in 4.9.3 +command -v inspect_artifacts >/dev/null 2>&1 && inspect_artifacts --all-packages || echo "inspect_artifacts needs conda-forge-ci-setup >=4.9.3" +( endgroup "Inspecting artifacts" ) 2> /dev/null diff --git a/.scripts/run_win_build.bat b/.scripts/run_win_build.bat index 721c3e03b3c54..be2f6f527d534 100644 --- a/.scripts/run_win_build.bat +++ b/.scripts/run_win_build.bat @@ -3,14 +3,53 @@ :: changes to this script, consider a proposal to conda-smithy so that other feedstocks can also :: benefit from the improvement. -:: Note: we assume a Miniforge installation is available - :: INPUTS (required environment variables) :: CONDA_BLD_PATH: path for the conda-build workspace :: CI: azure, or unset +:: MINIFORGE_HOME: where to install the base conda environment setlocal enableextensions enabledelayedexpansion +call :start_group "Provisioning build tools" +if "%MINIFORGE_HOME%"=="" set "MINIFORGE_HOME=%USERPROFILE%\Miniforge3" +:: Remove trailing backslash, if present +if "%MINIFORGE_HOME:~-1%"=="\" set "MINIFORGE_HOME=%MINIFORGE_HOME:~0,-1%" + +if exist "%MINIFORGE_HOME%\conda-meta\history" ( + echo Build tools already installed at %MINIFORGE_HOME%. +) else ( + where micromamba.exe >nul 2>nul + if !errorlevel! == 0 ( + set "MICROMAMBA_EXE=micromamba.exe" + echo "Found micromamba in PATH" + ) else ( + set "MAMBA_ROOT_PREFIX=!MINIFORGE_HOME!-micromamba-!RANDOM!" + set "MICROMAMBA_VERSION=1.5.10-0" + set "MICROMAMBA_URL=https://github.com/mamba-org/micromamba-releases/releases/download/!MICROMAMBA_VERSION!/micromamba-win-64" + set "MICROMAMBA_TMPDIR=!TMP!\micromamba-!RANDOM!" + set "MICROMAMBA_EXE=!MICROMAMBA_TMPDIR!\micromamba.exe" + + echo Downloading micromamba !MICROMAMBA_VERSION! + echo if not exist "!MICROMAMBA_TMPDIR!" mkdir "!MICROMAMBA_TMPDIR!" + echo certutil -urlcache -split -f "!MICROMAMBA_URL!" "!MICROMAMBA_EXE!" + if not exist "!MICROMAMBA_TMPDIR!" mkdir "!MICROMAMBA_TMPDIR!" + certutil -urlcache -split -f "!MICROMAMBA_URL!" "!MICROMAMBA_EXE!" + if !errorlevel! neq 0 exit /b !errorlevel! + ) + echo Creating environment + call "!MICROMAMBA_EXE!" create --yes --root-prefix "!MAMBA_ROOT_PREFIX!" --prefix "!MINIFORGE_HOME!" ^ + --channel conda-forge ^ + --file environment.yaml + if !errorlevel! neq 0 exit /b !errorlevel! + echo Moving pkgs cache from !MAMBA_ROOT_PREFIX! to !MINIFORGE_HOME! + move /Y "!MAMBA_ROOT_PREFIX!\pkgs" "!MINIFORGE_HOME!" >nul + if !errorlevel! neq 0 exit /b !errorlevel! + echo Removing !MAMBA_ROOT_PREFIX! + del /S /Q "!MAMBA_ROOT_PREFIX!" >nul + del /S /Q "!MICROMAMBA_TMPDIR!" >nul +) +call :end_group + call :start_group "Configuring conda" if "%CONDA_BLD_PATH%" == "" ( @@ -18,31 +57,30 @@ if "%CONDA_BLD_PATH%" == "" ( ) :: Activate the base conda environment -call activate base - -conda.exe config --set always_yes yes -if errorlevel 1 exit 1 -conda.exe config --set channel_priority strict -if errorlevel 1 exit 1 -conda.exe config --set solver libmamba -if errorlevel 1 exit 1 - -echo Installing dependencies -conda.exe install --file .\.ci_support\requirements.txt -if errorlevel 1 exit 1 +echo Activating "%MINIFORGE_HOME%" +call "%MINIFORGE_HOME%\Scripts\activate" :: Set basic configuration echo Setting up configuration +conda.exe config --env --set always_yes yes +if !errorlevel! neq 0 exit /b !errorlevel! +conda.exe config --env --set channel_priority strict +if !errorlevel! neq 0 exit /b !errorlevel! +conda.exe config --env --set solver libmamba +if !errorlevel! neq 0 exit /b !errorlevel! + setup_conda_rc .\ ".\recipes" .\.ci_support\%CONFIG%.yaml -if errorlevel 1 exit 1 +if !errorlevel! neq 0 exit /b !errorlevel! echo Run conda_forge_build_setup call run_conda_forge_build_setup -if errorlevel 1 exit 1 +if !errorlevel! neq 0 exit /b !errorlevel! -echo Force fetch origin/main -git fetch --force origin main:main -if errorlevel 1 exit 1 +if not "%CI%" == "" ( + echo Force fetch origin/main + git fetch --force origin main:main + if !errorlevel! neq 0 exit /b !errorlevel! +) echo Removing recipes also present in main cd recipes for /f "tokens=*" %%a in ('git ls-tree --name-only main -- .') do rmdir /s /q %%a && echo Removing recipe: %%a @@ -50,16 +88,25 @@ cd .. :: make sure there is a package directory so that artifact publishing works if not exist "%CONDA_BLD_PATH%\win-64\" mkdir "%CONDA_BLD_PATH%\win-64\" +if not exist "%CONDA_BLD_PATH%\win-arm64\" mkdir "%CONDA_BLD_PATH%\win-arm64\" +if not exist "%CONDA_BLD_PATH%\noarch\" mkdir "%CONDA_BLD_PATH%\noarch\" echo Index %CONDA_BLD_PATH% conda.exe index "%CONDA_BLD_PATH%" -if errorlevel 1 exit 1 +if !errorlevel! neq 0 exit /b !errorlevel! call :end_group echo Building all recipes -python .ci_support\build_all.py --arch 64 -if errorlevel 1 exit 1 +python .ci_support\build_all.py +if !errorlevel! neq 0 exit /b !errorlevel! + +call :start_group "Inspecting artifacts" + +:: inspect_artifacts was only added in conda-forge-ci-setup 4.6.0; --all-packages in 4.9.3 +WHERE inspect_artifacts >nul 2>nul && inspect_artifacts --all-packages || echo "inspect_artifacts needs conda-forge-ci-setup >=4.9.3" + +call :end_group exit diff --git a/LICENSE.txt b/LICENSE.txt index b470edf8dc82f..d5ba276d41cb7 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -24,4 +24,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README.md b/README.md index 62cbd1a891352..fa6886527e1f9 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,7 @@ This repo is a holding area for recipes destined for a conda-forge feedstock repo. To find out more about conda-forge, see https://github.com/conda-forge/conda-smithy. -[![Join the chat at https://gitter.im/conda-forge/conda-forge.github.io](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/conda-forge/conda-forge.github.io) - +[![Join the chat at https://conda-forge.zulipchat.com](https://img.shields.io/badge/Zulip-join_chat-53bfad.svg)](https://conda-forge.zulipchat.com) ## Feedstock conversion status @@ -11,24 +10,87 @@ This repo is a holding area for recipes destined for a conda-forge feedstock rep Failures with the above job are often caused by API rate limits from the various services used by conda-forge. This can result in empty feedstock repositories and will resolve itself automatically. -If the issue persists, support can be found [on Gitter](https://gitter.im/conda-forge/conda-forge.github.io). +If the issue persists, support can be found [on Zulip](https://conda-forge.zulipchat.com). ## Getting started 1. Fork this repository. -2. Make a new folder in `recipes` for your package. Look at the example recipe, our [documentation](http://conda-forge.org/docs/maintainer/adding_pkgs.html#) and the [FAQ](https://github.com/conda-forge/staged-recipes#faq) for help. +2. Make a new folder in `recipes` for your package. Look at the [example recipe](recipes/example), our [documentation](http://conda-forge.org/docs/maintainer/adding_pkgs.html#) and the [FAQ](https://github.com/conda-forge/staged-recipes#faq) for help. 3. Open a pull request. Building of your package will be tested on Windows, Mac and Linux. 4. When your pull request is merged a new repository, called a feedstock, will be created in the github conda-forge organization, and build/upload of your package will automatically be triggered. Once complete, the package is available on conda-forge. +### `pixi` + +`pixi` is a project based environment and task runner optimized for `conda`. Several of +the local workflows and their dependencies described below are captured in +`pixi.toml`. Install `pixi` via the [documented approaches](https://pixi.sh/latest/#installation), +or via `conda`/`mamba`/`micromamba`: + +```bash +$CONDA_EXE install -c conda-forge pixi +``` + +See the available tasks with `pixi task list`. + +## Local debugging with `build-locally.py` + +The script `build-locally.py` will guide you through the local debugging process. This script +will then launch the platform-specific scripts, which support some key environment variables in +macOS and Windows: + +- `MINIFORGE_HOME`: Where the build tools will be installed. Defaults to `~/Miniforge3`. +- `CONDA_BLD_PATH`: Where the build artifacts will be kept. Defaults to `~/Miniforge3/conda-bld` + on macOS and `C:\bld` on Windows. + +On Linux, everything runs in a Docker container. The `staged-recipes` directory is mounted as a volume. The resulting artifacts will be available under `build_artifacts` in the repository directory. + +`build-locally.py` can be run with any recent Python, or via a [`pixi`](#pixi) task: + +* `pixi run build-linux`: will launch a Docker container, provision all the necessary tools and build your recipe for Linux. +* `pixi run build-osx`: will provision a conda environment with the necessary tools to build your recipe for macOS. This involves fetching and caching the necessary Apple SDKs. +* `pixi run build-win`: will provision a conda environment with the necessary tools to build your recipe for Windows. + +These tasks will pass any extra arguments to `build-locally.py`, including `--help`. The resulting +artifacts will be available under `build_artifacts`. + +## Generating recipes with `grayskull` + +[grayskull](https://github.com/conda-incubator/grayskull) can generate recipes from +Python packages on [PyPI](https://pypi.org) or R packages on [CRAN](https://cran.r-project.org/). +The user should review the recipe generated, especially the license and dependencies. + +Use one of: -## Grayskull - recipe generator for Python packages on `pypi` +- manually + 1. install `grayskull`: `conda install -c conda-forge grayskull` + 2. generate recipe: + - `cd recipes && grayskull pypi PACKAGE_NAME_ON_PYPI_HERE [PACKAGE_NAME_ON_PYPI_HERE...]` + - `cd recipes && grayskull cran PACKAGE_NAME_ON_CRAN_HERE [PACKAGE_NAME_ON_CRAN_HERE...]` +- with [`pixi`](#pixi): + 1. generate recipe: + - `pixi run pypi PACKAGE_NAME_ON_PYPI_HERE [PACKAGE_NAME_ON_PYPI_HERE...]` + - `pixi run cran PACKAGE_NAME_ON_CRAN_HERE [PACKAGE_NAME_ON_CRAN_HERE...]` -For Python packages available on `pypi` it is possible to use [grayskull](https://github.com/conda-incubator/grayskull) to generate the recipe. The user should review the recipe generated, specially the license and dependencies. +## Linting recipes with `conda-smithy` -Installing `grayskull`: `conda install -c conda-forge grayskull` +The [`conda-smithy`](https://github.com/conda-forge/conda-smithy) package provides +helpful linters that can save CI resources by catching known issues up-front. -Generating recipe: `grayskull pypi PACKAGE_NAME_HERE` +Use one of: +- manually + - install `conda-smithy`: `conda install -c conda-forge conda-smithy` + - lint recipes: `conda-smithy recipe-lint --conda-forge recipes/*` +- with [`pixi`](#pixi): + - lint recipes: `pixi run lint` +> **NOTE** +> +> `conda-smithy` is +> [frequently updated](https://github.com/conda-forge/conda-smithy/blob/main/CHANGELOG.rst) +> with current best practices. Ensure using the latest with: +> +> - `$CONDA_EXE upgrade conda-smithy` +> - `pixi upgrade --feature conda-smithy` ## FAQ @@ -65,7 +127,7 @@ build: A full description of selectors is [in the conda docs](https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#preprocessing-selectors). -If the package can otherwise be `noarch` you can also skip it by using [virtual packages](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html). +If the package can otherwise be `noarch` you can also skip it by using [virtual packages](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html). _Note_: As the package will always be built on linux, it needs to be at least available on there. @@ -164,11 +226,11 @@ Then, a bot will label the PR as 'review-requested'. Due to GitHub limitations, first time contributors to conda-forge are unable to ping conda-forge teams directly, but you can [ask a bot to ping the team][1] using a special command in a comment on the PR to get the attention of the -`staged-recipes` team. You can also consider asking on our [Gitter channel][2] +`staged-recipes` team. You can also consider asking on our [Zulip chat][2] if your recipe isn't reviewed promptly. [1]: https://conda-forge.org/docs/maintainer/infrastructure.html#conda-forge-admin-please-ping-team -[2]: https://gitter.im/conda-forge/conda-forge.github.io +[2]: https://conda-forge.zulipchat.com All apologies in advance if your recipe PR does not receive prompt attention. This is a high volume repository and the reviewers are volunteers. Review times vary depending on the number of reviewers on a given language team and may be days or weeks. We are always @@ -181,5 +243,5 @@ please contact a member of @conda-forge/core. We'd love to have your help! There's no changelog file, but the following `git` command gives a good overview of the recent changes in the repository: ```bash -$ git log --merges -- ':!recipes' +$ git log --merges -- ':!recipes' ``` diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 32b938c3e80ec..69519a28ed7cc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,7 +9,33 @@ trigger: pr: - main -jobs: +stages: +- stage: Check + jobs: + - job: Skip + pool: + vmImage: 'ubuntu-22.04' + variables: + DECODE_PERCENTS: 'false' + RET: 'true' + steps: + - checkout: self + # We only need two: the PR merge commit, and the actual commit added by the user + fetchDepth: '2' + - bash: | + git_log=`git log --max-count=1 --skip=1 --pretty=format:"%B" | tr "\n" " "` + echo "##vso[task.setvariable variable=log]$git_log" + displayName: Obtain commit message + - bash: echo "##vso[task.setvariable variable=RET]false" + condition: or(contains(variables.log, '[skip azp]'), contains(variables.log, '[azp skip]'), contains(variables.log, '[skip ci]'), contains(variables.log, '[ci skip]')) + displayName: Skip build? + - bash: echo "##vso[task.setvariable variable=start_main;isOutput=true]$RET" + name: result + displayName: Export result +- stage: Build + condition: and(succeeded(), eq(dependencies.Check.outputs['Skip.result.start_main'], 'true')) + dependsOn: Check + jobs: - template: ./.azure-pipelines/azure-pipelines-linux.yml - template: ./.azure-pipelines/azure-pipelines-osx.yml - template: ./.azure-pipelines/azure-pipelines-win.yml diff --git a/broken-recipes/inplace-abn/build.sh b/broken-recipes/inplace-abn/build.sh new file mode 100755 index 0000000000000..ef63b4d336444 --- /dev/null +++ b/broken-recipes/inplace-abn/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -exo pipefail + +export TORCH_CUDA_ARCH_LIST="7.0;8.0;9.0" + +python -m pip install . -vv --no-deps --no-build-isolation diff --git a/broken-recipes/inplace-abn/meta.yaml b/broken-recipes/inplace-abn/meta.yaml new file mode 100644 index 0000000000000..98f5e01a139ce --- /dev/null +++ b/broken-recipes/inplace-abn/meta.yaml @@ -0,0 +1,87 @@ +{% set name = "inplace-abn" %} +{% set version = "1.1.0" %} + +{% if cuda_compiler_version in (None, "None", True, False) %} +{% set cuda_major = 0 %} +{% else %} +{% set cuda_major = environ.get("cuda_compiler_version", "11.8").split(".")[0] | int %} +{% endif %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/inplace-abn-{{ version }}.tar.gz + sha256: 7c286ca53d35cb37b0d1f8b9beb5a910c694129d3d4f7b8e53188370bb260c27 + +build: + number: 0 + skip: true # [osx] + skip: true # [win] + string: cpu_py{{ CONDA_PY }}_h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} # [cuda_compiler_version == "None"] + string: cuda{{ cuda_compiler_version | replace('.', '') }}_py{{ CONDA_PY }}_h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} # [cuda_compiler_version != "None"] + +requirements: + build: + - {{ compiler("cxx") }} + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] + - {{ stdlib("c") }} + {% if cuda_major >= 12 %} + - cuda-driver-dev # [build_platform != target_platform] + - cuda-cudart-dev # [build_platform != target_platform] + - cuda-nvrtc-dev # [build_platform != target_platform] + - cuda-nvtx-dev # [build_platform != target_platform] + - cuda-nvml-dev # [build_platform != target_platform] + - cuda-profiler-api # [build_platform != target_platform] + - libcublas-dev # [build_platform != target_platform] + - libcufft-dev # [build_platform != target_platform] + - libcurand-dev # [build_platform != target_platform] + - libcusolver-dev # [build_platform != target_platform] + - libcusparse-dev # [build_platform != target_platform] + {% endif %} + host: + # GPU requirements + - cudnn # [cuda_compiler_version != "None"] + - nccl # [cuda_compiler_version != "None"] + - magma # [cuda_compiler_version != "None"] + - cuda-version {{ cuda_compiler_version }} # [cuda_compiler_version != "None"] + {% if cuda_major >= 12 %} + - cuda-driver-dev + - cuda-cudart-dev + - cuda-nvrtc-dev + - cuda-nvtx-dev + - cuda-nvml-dev + - cuda-profiler-api + - libcublas-dev + - libcufft-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + {% endif %} + # other requirements + - pip + - python + - pytorch >=2.4.0 + - setuptools + - setuptools_scm + run: + - python + +test: + imports: + - inplace_abn + commands: + - pip check + requires: + - pip + +about: + home: https://github.com/mapillary/inplace_abn + summary: In-Place Activated BatchNorm for Memory-Optimized Training of DNNs + license: BSD-3-Clause + license_file: LICENSE + +extra: + recipe-maintainers: + - jeongseok-meta diff --git a/build-locally.py b/build-locally.py index 8b7434893256b..a43b1d7297a3d 100644 --- a/build-locally.py +++ b/build-locally.py @@ -6,9 +6,11 @@ import os import glob import subprocess +import sys from argparse import ArgumentParser import platform +BUILD_LOCALLY_FILTER = os.environ.get("BUILD_LOCALLY_FILTER", "*") def setup_environment(ns): os.environ["CONFIG"] = ns.config @@ -27,6 +29,10 @@ def setup_environment(ns): os.path.dirname(__file__), "SDKs" ) + # The default cache location might not be writable using docker on macOS. + if ns.config.startswith("linux") and platform.system() == "Darwin": + os.environ["CONDA_FORGE_DOCKER_RUN_ARGS"] = "-e RATTLER_CACHE_DIR=/tmp/rattler_cache" + def run_docker_build(ns): script = ".scripts/run_docker_build.sh" @@ -38,10 +44,17 @@ def run_osx_build(ns): subprocess.check_call([script]) +def run_win_build(ns): + script = ".scripts/run_win_build.bat" + subprocess.check_call(["cmd", "/D", "/Q", "/C", f"CALL {script}"]) + + def verify_config(ns): valid_configs = { - os.path.basename(f)[:-5] for f in glob.glob(".ci_support/*.yaml") + os.path.basename(f)[:-5] for f in glob.glob(f".ci_support/{BUILD_LOCALLY_FILTER}.yaml") } + if BUILD_LOCALLY_FILTER != "*": + print(f"filtering for '{BUILD_LOCALLY_FILTER}.yaml' configs") print(f"valid configs are {valid_configs}") if ns.config in valid_configs: print("Using " + ns.config + " configuration") @@ -54,18 +67,17 @@ def verify_config(ns): selections = list(enumerate(sorted(valid_configs), 1)) for i, c in selections: print(f"{i}. {c}") - s = input("\n> ") + try: + s = input("\n> ") + except KeyboardInterrupt: + print("\nno option selected, bye!", file=sys.stderr) + sys.exit(1) idx = int(s) - 1 ns.config = selections[idx][1] print(f"selected {ns.config}") else: raise ValueError("config " + ns.config + " is not valid") - # Remove the following, as implemented - if ns.config.startswith("win"): - raise ValueError( - f"only Linux/macOS configs currently supported, got {ns.config}" - ) - elif ns.config.startswith("osx") and platform.system() == "Darwin": + if ns.config.startswith("osx") and platform.system() == "Darwin": if "OSX_SDK_DIR" not in os.environ: raise RuntimeError( "Need OSX_SDK_DIR env variable set. Run 'export OSX_SDK_DIR=/opt'" @@ -95,6 +107,8 @@ def main(args=None): run_docker_build(ns) elif ns.config.startswith("osx"): run_osx_build(ns) + elif ns.config.startswith("win"): + run_win_build(ns) if __name__ == "__main__": diff --git a/conda_build_config.yaml b/conda_build_config.yaml deleted file mode 100644 index db8ca38cf404a..0000000000000 --- a/conda_build_config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: # [osx] - - 11.0 # [osx] \ No newline at end of file diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000000000..980fedaca3530 --- /dev/null +++ b/environment.yaml @@ -0,0 +1,16 @@ +name: build +channels: +- conda-forge +- nodefaults +dependencies: +- python 3.12.* +- conda >=24.9.2 +- conda-libmamba-solver >=24.9.0 +- conda-build >=24.9 +- conda-index >=0.3.0 +- conda-forge-ci-setup >=4.9.3,<5.0 +- conda-forge-pinning * +- frozendict * +- networkx 2.4.* +- rattler-build-conda-compat >=1.2.0,<2.0.0a0 + diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 0000000000000..8f0e81ff6ffd5 --- /dev/null +++ b/pixi.toml @@ -0,0 +1,86 @@ +# VVVVVV minimum `pixi` version +"$schema" = "https://pixi.sh/v0.36.0/schema/manifest/schema.json" + +[project] +name = "staged-recipes" +description = "The entry point to conda-forge" +version = "0.1.0" +license = "BSD-3-Clause" +license-file = "LICENSE.txt" +authors = ["@conda-forge/core"] +channels = ["conda-forge"] +platforms = [ + "linux-64", "linux-aarch64", "osx-64", "osx-arm64", "win-64" + # TODO: add here and below in `feature.win` when possible + # "win-arm64" +] + +[feature.python.dependencies] +python = "3.12.*" + +[feature.build.dependencies] +conda = ">=24.9.2" +conda-libmamba-solver = ">=24.9.0" +conda-build = ">=24.9" +conda-index = ">=0.3.0" +conda-forge-ci-setup = ">=4.9.3,<5.0" +conda-forge-pinning = "*" +frozendict = "*" +networkx = "2.4.*" +rattler-build-conda-compat = ">=1.2.0,<2.0.0a0" + +[feature.linux] +# The linux feature runs on every platform that can run Docker. +platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64", "win-64"] +[feature.linux.tasks.build-linux] +description = "build for Linux inside a docker container" +cmd = "python build-locally.py" +env = { BUILD_LOCALLY_FILTER = "linux*" } + +# The osx env will run conda-build directly on the machine. +[feature.osx] +platforms = ["osx-64", "osx-arm64"] +[feature.osx.tasks.build-osx] +description = "build directly on a MacOS host machine" +cmd = "python build-locally.py" +[feature.osx.tasks.build-osx.env] +BUILD_LOCALLY_FILTER = "osx*" +CONDA_BLD_PATH = "$PIXI_PROJECT_ROOT/build_artifacts" +MINIFORGE_HOME = "$CONDA_PREFIX" +OSX_SDK_DIR = "$PIXI_PROJECT_ROOT/.pixi/macOS-SDKs" + +# The windows env will run conda-build directly on the machine. +[feature.win] +platforms = ["win-64"] +[feature.win.tasks.build-win] +description = "build directly on a Windows host machine" +cmd = "python build-locally.py" +[feature.win.tasks.build-win.env] +BUILD_LOCALLY_FILTER = "win*" +CONDA_BLD_PATH = "$PIXI_PROJECT_ROOT/build_artifacts" +MINIFORGE_HOME = "$CONDA_PREFIX" + +[feature.grayskull.dependencies] +grayskull = ">=2.7.3" +[feature.grayskull.tasks.pypi] +description = "generate a recipe for a PyPI package name with `grayskull`" +cmd = "cd recipes && grayskull pypi --strict-conda-forge" +[feature.grayskull.tasks.cran] +description = "generates a recipe for a CRAN package name with `grayskull`" +cmd = "cd recipes && grayskull cran --strict-conda-forge" + +[feature.conda-smithy.dependencies] +conda-smithy = ">=3.44.6,<4" +[feature.conda-smithy.tasks.lint] +description = "validate all recipes with `conda-smithy`" +cmd = "conda-smithy recipe-lint --conda-forge recipes/*" + +[environments] +# linux only needs Python to run build-locally.py. Everything else happens in Docker. +linux = ["linux", "python"] +# osx and win do run natively on the machine, so we need the build deps too +osx = ["osx", "python", "build"] +win = ["win", "python", "build"] +build = ["python", "build"] +grayskull = ["python", "grayskull"] +conda-smithy = ["python", "conda-smithy"] diff --git a/recipes/example-v1/README.md b/recipes/example-v1/README.md new file mode 100644 index 0000000000000..f2a0e51059f33 --- /dev/null +++ b/recipes/example-v1/README.md @@ -0,0 +1,5 @@ +# V1 recipe format + +This recipe is an example of the v1 recipe format that was defined by [CEP 13](https://github.com/conda/ceps/blob/main/cep-13.md). The v1 recipe format is fully functional when built with rattler-build, but is not yet fully supported by conda-forge's automation. + +See https://github.com/conda-forge/conda-forge.github.io/issues/2308 for progress on general support for this new format. diff --git a/recipes/example-v1/recipe.yaml b/recipes/example-v1/recipe.yaml new file mode 100644 index 0000000000000..531e95ca07e3d --- /dev/null +++ b/recipes/example-v1/recipe.yaml @@ -0,0 +1,102 @@ +# This example shows how to define a recipe using the new YAML based recipe format introduced by +# CEP 13. + +# For more information about this format see: https://prefix-dev.github.io/rattler-build/latest/reference/recipe_file/ + +# The main differences with the old format is that no preprocessing is required for the file to be valid YAML. +# This means: +# - No "selectors", use YAML if-then-else expressions instead (https://prefix-dev.github.io/rattler-build/latest/selectors/) +# - Jinja expressions are formatted with a leading `$` to make them valid YAML + +# Note: there are many handy hints in comments in this example -- remove them when you've finalized your recipe + +# Define variables in this section that you can use in other parts. +context: + name: simplejson + version: "3.8.2" + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: https://pypi.org/packages/source/${{ name[0] }}/${{ name }}/${{ name }}-${{ version }}.tar.gz + # If getting the source from GitHub, remove the line above, + # uncomment the line below, and modify as needed. Use releases if available: + # url: https://github.com/simplejson/simplejson/releases/download/${{ version }}/simplejson-${{ version }}.tar.gz + # and otherwise fall back to archive: + # url: https://github.com/simplejson/simplejson/archive/v${{ version }}.tar.gz + sha256: d58439c548433adcda98e695be53e526ba940a4b9c44fb9a05d92cd495cdd47f + # sha256 is the preferred checksum -- you can get it for a file with: + # `openssl sha256 `, + # or, in one go without keeping the file with: + # `curl -L | sha256 + # You may need the packages or openssl and/or curl, available on conda-forge: + # `conda install openssl curl -c conda-forge`` + +build: + # Uncomment the following line if the package is pure Python and the recipe is exactly the same for all platforms. + # It is okay if the dependencies are not built for all platforms/versions, although selectors are still not allowed. + # See https://conda-forge.org/docs/maintainer/knowledge_base.html#noarch-python for more details. + # noarch: python + # If the installation is complex, or different between Unix and Windows, use separate build.bat and build.sh files instead of this key. + # By default, the package will be built for the Python versions supported by conda-forge and for all major OSs. + # Uncomment the following lines to limit to Python 3.5 and newer (for example) + # skip: + # - match(python, "<3.5") + # or the following to limit to Windows. + # skip: + # - win + script: python -m pip install . -vv + number: 0 + +requirements: + build: + # If your project compiles code (such as a C extension) then add the required compilers as separate entries here. + # Compiler names include 'c', 'cxx' and 'fortran', among others. + - ${{ compiler('c') }} + - ${{ stdlib('c') }} # If you need any compiler, add the C standard library ("stdlib") too + host: + - python + - pip + - setuptools + run: + - python + +tests: + # Some packages might need a `test/commands` key to check CLI. + - python: + # A list of modules that the test will try to import + imports: + - simplejson + - simplejson.tests + # Also run `pip check` to verify the integrity + pip_check: true + +about: + homepage: https://github.com/simplejson/simplejson + summary: 'Simple, fast, extensible JSON encoder/decoder for Python' + description: | + simplejson is a simple, fast, complete, correct and extensible + JSON encoder and decoder for Python 2.5+ and + Python 3.3+. It is pure Python code with no dependencies, but includes + an optional C extension for a serious speed boost. + # Remember to specify the license variants for BSD, Apache, GPL, and LGPL. + # Use the SPDX identifier, e.g: GPL-2.0-only instead of GNU General Public License version 2.0 + # See https://spdx.org/licenses/ + license: MIT + # It is required to include a license file in the package, + # (even if the license doesn't require it) using the license_file entry. + # Please also note that some projects have multiple license files which all need to be added using a valid yaml list. + # See https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#license-file + license_file: LICENSE.txt + # The documentation and repository URLs are optional. + documentation: https://simplejson.readthedocs.io/ + repository: https://github.com/simplejson/simplejson + +extra: + recipe-maintainers: + # GitHub IDs for maintainers of the recipe. + # Always check with the people listed below if they are OK becoming maintainers of the recipe. (There will be spam!) + - LisaSimpson + - LandoCalrissian diff --git a/recipes/example/meta.yaml b/recipes/example/meta.yaml index 66aa12c04c2b9..f095288e58d10 100644 --- a/recipes/example/meta.yaml +++ b/recipes/example/meta.yaml @@ -13,7 +13,7 @@ package: version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz + url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz # If getting the source from GitHub, remove the line above, # uncomment the line below, and modify as needed. Use releases if available: # url: https://github.com/simplejson/simplejson/releases/download/{{ version }}/simplejson-{{ version }}.tar.gz @@ -41,11 +41,14 @@ build: requirements: build: # If your project compiles code (such as a C extension) then add the required compilers as separate entries here. - # Compilers are named 'c', 'cxx' and 'fortran'. + # Compilers are named 'c', 'cxx', 'fortran', among others - {{ compiler('c') }} + # this is necessary for all compiled recipes + - {{ stdlib('c') }} host: - python - pip + - setuptools run: - python diff --git a/recipes/robotframework-mqttlibrary/meta.yaml b/recipes/robotframework-mqttlibrary/meta.yaml deleted file mode 100644 index c85408c07cdc1..0000000000000 --- a/recipes/robotframework-mqttlibrary/meta.yaml +++ /dev/null @@ -1,43 +0,0 @@ -{% set name = "robotframework-mqttlibrary" %} -{% set version = "0.7.1" %} - -package: - name: {{ name|lower }} - version: {{ version }} - -source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/robotframework-mqttlibrary-{{ version }}.tar.gz - sha256: b40d426cb33242b05b7f03a32f56bcb0c601f4568553a09dbbbcff17d0e900de - -build: - noarch: python - script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation - number: 0 - -requirements: - host: - - pip - - python >=3.6 - run: - - python >=3.6 - - robotframework - - paho-mqtt - -test: - imports: - - MQTTLibrary - commands: - - pip check - requires: - - pip - -about: - home: https://github.com/randomsync/robotframework-mqttlibrary - summary: MQTT Keyword Library Robot Framework - doc_url: https://pythonhosted.org/robotframework-mqttlibrary/ - license: Apache-2.0 - license_file: LICENSE.txt - -extra: - recipe-maintainers: - - nisharai1 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e14b7619292fb..0000000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[flake8] -max-line-length=88