Return nested recipe's dataframe from the recipe write connector #145

Workflow file for this run

	name: CI

	on:
	# Push is limited to the branches that publish an image. A branch with an open
	# PR in this repo would otherwise produce a push run and a pull_request run for
	# the same commit, and the two would cancel each other out of the concurrency
	# group below. Feature branches are covered by their pull_request run.
	push:
	branches: [ main, dev ]
	pull_request:
	branches: [ main, dev ]

	concurrency:
	group: ci-${{ github.event_name }}-${{ github.event.pull_request.number \|\| github.ref_name }}
	# Pushes to main/dev publish and promote :latest / :dev, so they must always
	# finish. Only superseded PR runs are safe to cancel.
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	env:
	REGISTRY: ghcr.io
	IMAGE_NAME: wrangleworks/wrangles

	jobs:
	config:
	runs-on: ubuntu-latest
	outputs:
	matrix: ${{ steps.resolve.outputs.matrix }}
	build_tag: ${{ steps.resolve.outputs.build_tag }}
	release_tag: ${{ steps.resolve.outputs.release_tag }}
	push_image: ${{ steps.resolve.outputs.push_image }}
	run_build: ${{ steps.resolve.outputs.run_build }}
	steps:
	- id: resolve
	env:
	BASE_REF: ${{ github.base_ref }}
	EVENT_NAME: ${{ github.event_name }}
	run: \|
	FULL=false
	[[ "$GITHUB_REF_NAME" == "main" \|\| "$BASE_REF" == "main" ]] && FULL=true
	if [[ "$FULL" == "true" ]]; then
	echo 'matrix={"os":["ubuntu-latest","windows-latest"],"python-version":["3.11","3.13"]}' >> "$GITHUB_OUTPUT"
	else
	echo 'matrix={"os":["ubuntu-latest"],"python-version":["3.11"]}' >> "$GITHUB_OUTPUT"
	fi

	RELEASE_TAG=""
	PUSH_IMAGE=false
	RUN_BUILD=false

	if [[ "$EVENT_NAME" == "pull_request" ]]; then
	RUN_BUILD=true
	elif [[ "$EVENT_NAME" == "push" ]]; then
	if [[ "$GITHUB_REF_NAME" == "main" ]]; then
	RELEASE_TAG="latest"
	PUSH_IMAGE=true
	RUN_BUILD=true
	elif [[ "$GITHUB_REF_NAME" == "dev" ]]; then
	RELEASE_TAG="dev"
	PUSH_IMAGE=true
	RUN_BUILD=true
	fi
	fi

	# Pushes land on an immutable per-commit tag. The mutable test-image tag
	# (dev / latest) is only moved onto that image once test-container
	# passes, so validation jobs never resolve it to an untested build.
	if [[ "$PUSH_IMAGE" == "true" ]]; then
	BUILD_TAG="sha-${GITHUB_SHA}"
	else
	BUILD_TAG="ci-${GITHUB_SHA}"
	fi

	echo "build_tag=${BUILD_TAG}" >> "$GITHUB_OUTPUT"
	echo "release_tag=${RELEASE_TAG}" >> "$GITHUB_OUTPUT"
	echo "push_image=${PUSH_IMAGE}" >> "$GITHUB_OUTPUT"
	echo "run_build=${RUN_BUILD}" >> "$GITHUB_OUTPUT"

	pytest:
	needs: [config]
	runs-on: ${{ matrix.os }}
	strategy:
	matrix: ${{ fromJSON(needs.config.outputs.matrix) }}
	permissions:
	contents: read
	steps:
	- name: Checkout Repository
	uses: actions/checkout@v5

	- name: Set up Python
	uses: actions/setup-python@v6
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install Dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install pytest==9.0.2 lorem pytest-mock
	pip install -r requirements-full.txt
	pip install .

	- name: Run Tests
	run: pytest
	env:
	WRANGLES_USER: ${{ secrets.WRANGLES_USER }}
	WRANGLES_PASSWORD: ${{ secrets.WRANGLES_PASSWORD }}
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	HUGGINGFACE_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
	OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
	SERPAPI_API_KEY: ${{ secrets.SERPAPI_API_KEY }}
	GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}

	test-pip-install:
	needs: [config]
	runs-on: ${{ matrix.os }}
	strategy:
	matrix: ${{ fromJSON(needs.config.outputs.matrix) }}
	permissions:
	contents: read
	steps:
	- name: Checkout Repository
	uses: actions/checkout@v5

	- name: Set up Python
	uses: actions/setup-python@v6
	with:
	python-version: ${{ matrix.python-version }}

	- name: Test pip install
	run: \|
	python -m pip install --upgrade pip
	pip install .
	wrangles.recipe tests/samples/generate-data.wrgl.yml

	test-generate-schema:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Repository
	uses: actions/checkout@v5

	- name: Set up Python
	uses: actions/setup-python@v6
	with:
	python-version: '3.12'

	- name: Install Dependencies
	run: \|
	pip install -r requirements-full.txt
	pip install jsonschema

	- name: Generate and Test Schema
	run: cd schema && python generate_recipe_schema.py

	- name: Save Schema as Artifact
	uses: actions/upload-artifact@v6
	with:
	name: schema
	path: schema/schema.json

	build:
	runs-on: ubuntu-latest
	needs: [config, pytest, test-pip-install, test-generate-schema]
	if: needs.config.outputs.run_build == 'true'
	permissions:
	contents: read
	packages: write
	outputs:
	digest: ${{ steps.build.outputs.digest }}
	steps:
	- name: Checkout Repository
	uses: actions/checkout@v5

	# Login against a Docker registry only when pushing
	# https://github.com/docker/login-action
	- name: Log into registry ${{ env.REGISTRY }}
	if: needs.config.outputs.push_image == 'true'
	uses: docker/login-action@v4
	with:
	registry: ${{ env.REGISTRY }}
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	# Extract metadata (tags, labels) for Docker
	# https://github.com/docker/metadata-action
	- name: Extract Docker metadata
	id: meta
	uses: docker/metadata-action@v6
	with:
	images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

	# Build and push the CI test image with Buildx (build only, never push, on PRs)
	# https://github.com/docker/build-push-action
	- name: Build and push CI test image
	id: build
	uses: docker/build-push-action@v7
	with:
	context: .
	load: ${{ needs.config.outputs.push_image != 'true' }}
	push: ${{ needs.config.outputs.push_image == 'true' }}
	tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.config.outputs.build_tag }}
	labels: ${{ steps.meta.outputs.labels }}

	- name: Validate pull request test image
	if: needs.config.outputs.push_image != 'true'
	env:
	IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.config.outputs.build_tag }}
	run: \|
	docker run --rm \
	--volume "${GITHUB_WORKSPACE}:/workspace:ro" \
	"$IMAGE" \
	python /workspace/scripts/container_smoke.py
	docker run --rm "$IMAGE" python -m pip check
	docker run --rm "$IMAGE" python -m pip freeze
	docker run --rm \
	--volume "${GITHUB_WORKSPACE}:/workspace:ro" \
	--workdir /workspace/tests/samples \
	"$IMAGE" \
	sh -c 'wrangles.recipe recipe-basic.wrgl.yml && wrangles.recipe recipe_custom_function.wrgl.yml -f custom_functions.py'

	test-container:
	runs-on: ubuntu-latest
	if: needs.config.outputs.push_image == 'true'
	needs: [config, build]
	container:
	image: ghcr.io/wrangleworks/wrangles:${{ needs.config.outputs.build_tag }}
	env:
	WRANGLES_USER: ${{ secrets.WRANGLES_USER }}
	WRANGLES_PASSWORD: ${{ secrets.WRANGLES_PASSWORD }}
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	HUGGINGFACE_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
	OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
	SERPAPI_API_KEY: ${{ secrets.SERPAPI_API_KEY }}
	GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
	steps:
	- name: Checkout Repository
	uses: actions/checkout@v5

	- name: Remove wrangles folder
	run: rm -r wrangles

	- name: Validate CI test runtime
	run: \|
	python scripts/container_smoke.py
	python -m pip check
	python -m pip freeze

	- name: Install Test Dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install pytest==9.0.2 lorem pytest-mock

	- name: Run Tests
	run: pytest

	- name: Run Recipe (Remote)
	run: wrangles.recipe 42f319a8-0849-4177

	- name: Run Recipe (Local)
	run: >-
	cd tests/samples
	&& wrangles.recipe recipe-basic.wrgl.yml
	&& wrangles.recipe recipe_custom_function.wrgl.yml -f custom_functions.py

	# Moves :dev / :latest onto the CI test image that test-container validated.
	# Retagging by digest guarantees the promoted image is the tested one and not
	# a rebuild, so no consumer of a mutable tag ever sees an untested image.
	promote-image:
	name: Promote CI Test Image
	runs-on: ubuntu-latest
	if: needs.config.outputs.push_image == 'true'
	needs: [config, build, test-container]
	permissions:
	contents: read
	packages: write
	steps:
	- name: Log into registry ${{ env.REGISTRY }}
	uses: docker/login-action@v4
	with:
	registry: ${{ env.REGISTRY }}
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Retag tested image as ${{ needs.config.outputs.release_tag }}
	run: \|
	docker buildx imagetools create \
	--tag "${REGISTRY}/${IMAGE_NAME}:${RELEASE_TAG}" \
	"${REGISTRY}/${IMAGE_NAME}@${DIGEST}"
	env:
	RELEASE_TAG: ${{ needs.config.outputs.release_tag }}
	DIGEST: ${{ needs.build.outputs.digest }}

Navigation Menu

Return nested recipe's dataframe from the recipe write connector #145

Workflow file

Return nested recipe's dataframe from the recipe write connector #145

Uh oh!

Workflow file for this run