From 70004a6d6db178650a23412da358ccff405e449d Mon Sep 17 00:00:00 2001 From: JonSnow1807 Date: Tue, 29 Jul 2025 19:43:06 -0400 Subject: [PATCH 1/2] Add complete Colima support for metaflow-dev - Add Docker context management to properly route commands through Colima - Configure required environment variables (DOCKER_HOST, etc.) - Add setup script for easy Colima configuration - Support both Docker Desktop and Colima transparently - Add comprehensive documentation for Colima usage - Maintain backward compatibility with Docker Desktop This enables users who cannot use Docker Desktop due to licensing to use Colima as a drop-in replacement for Metaflow development. Fixes #2362 --- .gitignore | 4 ++ README.md | 7 +++ devtools/Makefile | 52 ++++++++++++++++--- devtools/setup-colima.sh | 108 +++++++++++++++++++++++++++++++++++++++ docs/colima.md | 65 +++++++++++++++++++++++ 5 files changed, 229 insertions(+), 7 deletions(-) create mode 100755 devtools/setup-colima.sh create mode 100644 docs/colima.md diff --git a/.gitignore b/.gitignore index b36301c7c44..8afc8b0bb6e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ stubs/version.py # devtools .devtools + +# Colima environment files +.env.colima +activate-colima.sh \ No newline at end of file diff --git a/README.md b/README.md index b7bb74cdf8d..5eadd4df2f4 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,13 @@ For more details on Metaflow’s features and best practices, check out: If you need help, don’t hesitate to reach out on our [Slack community](http://slack.outerbounds.co/)! +### Docker Runtime + +Metaflow development requires Docker. You can use either: +- **Docker Desktop** (default) +- **[Colima](docs/colima.md)** (open-source alternative for macOS) + +Check your Docker setup with: `make check-docker` ### Deploying infrastructure for Metaflow in your cloud diff --git a/devtools/Makefile b/devtools/Makefile index c30db2a26bd..b66d0fb31ef 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -67,17 +67,55 @@ install-helm: fi check-docker: - @if ! command -v docker >/dev/null 2>&1; then \ - echo "❌ Docker is not installed. Please install Docker first: https://docs.docker.com/get-docker/"; \ + @echo "Checking Docker runtime..." + @if command -v docker >/dev/null 2>&1; then \ + if docker info >/dev/null 2>&1; then \ + if colima status 2>&1 | grep -q "is running"; then \ + echo "✓ Using Colima as Docker runtime"; \ + $(MAKE) configure-colima; \ + elif docker context ls | grep -q "desktop-linux"; then \ + echo "✓ Using Docker Desktop"; \ + else \ + echo "✓ Using generic Docker runtime"; \ + fi; \ + else \ + if command -v colima >/dev/null 2>&1; then \ + echo "❌ Docker daemon not running. Colima is installed but not started."; \ + echo " Run: colima start"; \ + else \ + echo "❌ Docker daemon not running."; \ + echo " Please start Docker Desktop or install and start Colima:"; \ + echo " brew install colima && colima start"; \ + fi; \ + exit 1; \ + fi; \ + else \ + echo "❌ Docker CLI not found. Please install Docker or Colima."; \ exit 1; \ fi - @echo "🔍 Checking Docker daemon..." - @if [ "$(shell uname)" = "Darwin" ]; then \ - open -a Docker || (echo "❌ Please start Docker Desktop" && exit 1); \ + +configure-colima: + @if [ ! -f .env.colima ]; then \ + echo "export DOCKER_HOST=\"unix://\$${HOME}/.colima/default/docker.sock\"" > .env.colima; \ + echo "export DOCKER_BUILDKIT=1" >> .env.colima; \ + echo "export COMPOSE_DOCKER_CLI_BUILD=1" >> .env.colima; \ + fi + @if DOCKER_HOST="unix://\$${HOME}/.colima/default/docker.sock" docker version >/dev/null 2>&1; then \ + echo "✓ Docker commands routing through Colima successfully"; \ else \ - docker info >/dev/null 2>&1 || (echo "❌ Docker daemon is not running." && exit 1); \ + echo "❌ Failed to connect to Docker through Colima"; \ + exit 1; \ fi - @echo "✅ Docker is running" + +-include .env.colima + +ifdef DOCKER_HOST + DOCKER_CMD := DOCKER_HOST=$(DOCKER_HOST) docker + DOCKER_COMPOSE_CMD := DOCKER_HOST=$(DOCKER_HOST) docker-compose +else + DOCKER_CMD := docker + DOCKER_COMPOSE_CMD := docker-compose +endif install-brew: @if [ "$(shell uname)" = "Darwin" ] && ! command -v brew >/dev/null 2>&1; then \ diff --git a/devtools/setup-colima.sh b/devtools/setup-colima.sh new file mode 100755 index 00000000000..48db00656ec --- /dev/null +++ b/devtools/setup-colima.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# setup-colima.sh - Complete Colima setup for Metaflow development + +set -e + +COLIMA_PROFILE="${COLIMA_PROFILE:-default}" +COLIMA_CPUS="${COLIMA_CPUS:-4}" +COLIMA_MEMORY="${COLIMA_MEMORY:-8}" +COLIMA_DISK="${COLIMA_DISK:-60}" + +echo "Setting up Colima for Metaflow development..." + +# Check if Colima is installed +if ! command -v colima &> /dev/null; then + echo "❌ Colima not found. Installing via Homebrew..." + brew install colima +fi + +# Check if Docker CLI is installed +if ! command -v docker &> /dev/null; then + echo "❌ Docker CLI not found. Installing..." + brew install docker +fi + +# Check if Docker Compose is installed +if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then + echo "⚠️ Docker Compose not found. Installing..." + brew install docker-compose +fi + +# Stop existing Colima instance if running +if colima status &> /dev/null; then + echo "Stopping existing Colima instance..." + colima stop +fi + +# Start Colima with appropriate resources for Metaflow +echo "Starting Colima with optimized settings for Metaflow..." +echo " CPUs: ${COLIMA_CPUS}" +echo " Memory: ${COLIMA_MEMORY}GB" +echo " Disk: ${COLIMA_DISK}GB" + +colima start \ + --cpu "${COLIMA_CPUS}" \ + --memory "${COLIMA_MEMORY}" \ + --disk "${COLIMA_DISK}" \ + --vm-type vz \ + --network-address + +# Wait for Colima to be ready +echo "Waiting for Colima to be ready..." +for i in {1..30}; do + if docker version &> /dev/null; then + break + fi + sleep 1 +done + +# Set up Docker context +echo "Configuring Docker context..." +docker context use colima 2>/dev/null || { + echo "Creating colima context..." + docker context create colima \ + --description "Colima Docker Desktop replacement" \ + --docker "host=unix://${HOME}/.colima/${COLIMA_PROFILE}/docker.sock" + docker context use colima +} + +# Create environment file +echo "Creating environment configuration..." +cat > .env.colima < activate-colima.sh <<'EOF' +#!/bin/bash +# Source this file to activate Colima environment +# Usage: source activate-colima.sh + +if [ -f .env.colima ]; then + source .env.colima + echo "✓ Colima environment activated" + echo " DOCKER_HOST=${DOCKER_HOST}" +else + echo "❌ .env.colima not found. Run setup-colima.sh first." +fi +EOF +chmod +x activate-colima.sh + +# Test the setup +echo "Testing Docker connectivity..." +if docker run --rm hello-world &> /dev/null; then + echo "✓ Docker is working correctly through Colima" +else + echo "❌ Docker test failed" + exit 1 +fi + +echo "" +echo "✅ Colima setup complete!" +echo "" +echo "To use Colima in your current shell, run:" +echo " source activate-colima.sh" \ No newline at end of file diff --git a/docs/colima.md b/docs/colima.md new file mode 100644 index 00000000000..15da6832d23 --- /dev/null +++ b/docs/colima.md @@ -0,0 +1,65 @@ +# Using Colima with Metaflow Development + +Colima is an open-source alternative to Docker Desktop for macOS. + +## Installation + +```bash +brew install colima docker docker-compose +``` + +## Quick Setup + +Run the setup script: +```bash +./devtools/setup-colima.sh +``` + +Then activate Colima in your shell: +```bash +source activate-colima.sh +``` + +## Manual Setup + +If you prefer to set up manually: + +```bash +# Start Colima +colima start --cpu 4 --memory 8 --disk 60 + +# Use Colima context +docker context use colima + +# Set environment variables +export DOCKER_HOST="unix://${HOME}/.colima/default/docker.sock" +``` + +## Verify Setup + +```bash +make check-docker +``` + +You should see: "✓ Using Colima as Docker runtime" + +## Switching Between Docker Desktop and Colima + +To use Colima: +```bash +docker context use colima +source activate-colima.sh +``` + +To use Docker Desktop: +```bash +docker context use desktop-linux +unset DOCKER_HOST +``` + +## Troubleshooting + +If Docker commands fail: +1. Check Colima is running: `colima status` +2. Start if needed: `colima start` +3. Re-run setup: `./devtools/setup-colima.sh` \ No newline at end of file From a65bdc96f192c03c7101b68272f8437b4c8b99bc Mon Sep 17 00:00:00 2001 From: JonSnow1807 Date: Tue, 29 Jul 2025 20:27:07 -0400 Subject: [PATCH 2/2] Fix: Ensure Docker environment variables are exported and improve macOS compatibility - Export DOCKER_HOST and related variables for sub-processes - Make vm-type conditional based on macOS version (vz requires macOS 13+) - Ensures Colima configuration is properly used throughout the build process --- devtools/Makefile | 7 +++++++ devtools/setup-colima.sh | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/devtools/Makefile b/devtools/Makefile index b66d0fb31ef..edc92d0a52b 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -107,6 +107,13 @@ configure-colima: exit 1; \ fi +# Export for sub-makes +ifdef DOCKER_HOST +export DOCKER_HOST +export DOCKER_BUILDKIT +export COMPOSE_DOCKER_CLI_BUILD +endif + -include .env.colima ifdef DOCKER_HOST diff --git a/devtools/setup-colima.sh b/devtools/setup-colima.sh index 48db00656ec..b75fd1b5816 100755 --- a/devtools/setup-colima.sh +++ b/devtools/setup-colima.sh @@ -40,11 +40,18 @@ echo " CPUs: ${COLIMA_CPUS}" echo " Memory: ${COLIMA_MEMORY}GB" echo " Disk: ${COLIMA_DISK}GB" +# Detect macOS version for vm-type +if [[ $(uname) == "Darwin" ]] && [[ $(sw_vers -productVersion | cut -d. -f1) -ge 13 ]]; then + VM_TYPE="--vm-type vz" +else + VM_TYPE="" +fi + colima start \ --cpu "${COLIMA_CPUS}" \ --memory "${COLIMA_MEMORY}" \ --disk "${COLIMA_DISK}" \ - --vm-type vz \ + ${VM_TYPE} \ --network-address # Wait for Colima to be ready