add cross compilation 🥴

kp2pml30 · kp2pml30 · commit 7a6de196b4fa · 2024-11-14T12:26:53.000+04:00
diff --git a/.ci/executor-target-linux-aarch64.rb b/.ci/executor-target-linux-aarch64.rb
@@ -0,0 +1,3 @@
+extend_config({
+	:executor_target => "aarch64-unknown-linux-gnu"
+})
diff --git a/.github/workflows/genvm-all.yaml b/.github/workflows/genvm-all.yaml
@@ -112,8 +112,9 @@ jobs:
     needs: [pre-commit]
     uses: ./.github/workflows/genvm-build-template.yaml
     with:
-      build_type: debug
       runs_on: ubuntu-latest
+      preloads: ""
+      install_also: ""
     secrets: inherit
 
 #  build-genvm-release-ubuntu-latest:
diff --git a/.github/workflows/genvm-build-template.yaml b/.github/workflows/genvm-build-template.yaml
@@ -2,10 +2,14 @@ name: GenVM build GenVM template
 on:
   workflow_call:
     inputs:
-      build_type:
+      preloads:
         type: string
         required: true
-        description: build type
+        description: preloads
+      install_also:
+        type: string
+        required: true
+        description: install_also
       runs_on:
         type: string
         required: true
@@ -31,13 +35,13 @@ jobs:
       - name: Get source
         uses: ./.github/actions/get-src
         with:
-          install_also: --genvm --rust
+          install_also: --genvm --rust ${{ inputs.install_also }}
           third_party: executor/third-party/wasmtime executor/third-party/wasm-tools
       - name: install tools and configure
         run: |
           cd "$GITHUB_WORKSPACE" && \
           source env.sh && \
-          ./tools/ya-build/ya-build config --preload .ci/${{ inputs.build_type }}-conf.rb
+          ./tools/ya-build/ya-build config ${{ inputs.preloads }}
       - name: build
         run: |
           cd "$GITHUB_WORKSPACE" && \
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -22,18 +22,28 @@ jobs:
     uses: ./.github/workflows/genvm-build-template.yaml
     secrets: inherit
     with:
-      build_type: release
       runs_on: ubuntu-latest
+      preloads: --preload .ci/release-conf.rb
+      install_also: ""
+
+  release-build-genvm-release-ubuntu-latest-aarch64:
+    uses: ./.github/workflows/genvm-build-template.yaml
+    secrets: inherit
+    with:
+      runs_on: ubuntu-latest
+      preloads: --preload .ci/release-conf.rb --preload .ci/executor-target-linux-aarch64.rb
+      install_also: --cross-linux-aarch64
 
   release-build-genvm-release-macos-latest:
     uses: ./.github/workflows/genvm-build-template.yaml
     secrets: inherit
     with:
-      build_type: release
       runs_on: macos-latest
+      preloads: --preload .ci/release-conf.rb
+      install_also: ""
 
   release-publish:
-    needs: [build-runners, release-build-genvm-release-ubuntu-latest, release-build-genvm-release-macos-latest]
+    needs: [build-runners, release-build-genvm-release-ubuntu-latest, release-build-genvm-release-macos-latest, release-build-genvm-release-ubuntu-latest-aarch64]
     runs-on: ubuntu-latest
     permissions:
       contents: write  # Needed for creating releases
@@ -45,6 +55,8 @@ jobs:
           mv ${{ needs.release-build-genvm-release-ubuntu-latest.outputs.artifact_name }} genvm-linux-amd64.zip && \
           wget -q ${{ needs.release-build-genvm-release-macos-latest.outputs.artifact_url }} && \
           mv ${{ needs.release-build-genvm-release-macos-latest.outputs.artifact_name }} genvm-macos-arm64.zip && \
+          wget -q ${{ needs.release-build-genvm-release-ubuntu-latest-aarch64.outputs.artifact_url }} && \
+          mv ${{ needs.release-build-genvm-release-ubuntu-latest-aarch64.outputs.artifact_name }} genvm-linux-arm64.zip && \
           true
 
       - name: Create Release
@@ -54,6 +66,7 @@ jobs:
           files: |
             genvm-runners.zip
             genvm-linux-amd64.zip
+            genvm-linux-arm64.zip
             genvm-macos-arm64.zip
           name: Release ${{ github.ref_name }}
           draft: false
diff --git a/build-scripts/install/install-deps.rb b/build-scripts/install/install-deps.rb
@@ -22,6 +22,8 @@
 OptionParser.new do |opts|
 	opts.on '--genvm'
 	opts.on '--rust'
+	opts.on '--cross-linux-aarch64'
+	opts.on '--zig'
 	opts.on '--rust-det'
 	opts.on '--os'
 	opts.on '--wasi'
@@ -90,6 +92,7 @@ def find_executable(name)
 end.call()
 
 logger.info("detected target is #{TARGET_TRIPLE}")
+$logger = logger
 
 OS = (Proc.new {
 	re = {
@@ -127,6 +130,7 @@ def find_executable(name)
 
 download_dir = root.join('tools', 'downloaded')
 download_dir.mkpath()
+$download_dir = download_dir
 
 logger.debug("download dir is #{download_dir}")
 
@@ -148,6 +152,9 @@ def load_packages_from_lists(dir)
 
 if options[:os]
 	load_packages_from_lists 'os'
+	#if options[:'cross-linux-aarch64']
+	#	load_packages_from_lists 'os-linux-aarch64'
+	#end
 end
 
 if not RUBY_VERSION =~ /^3\./
@@ -166,6 +173,27 @@ def load_packages_from_lists(dir)
 	raise "rustup not found" if rustup.nil?
 end
 
+if options[:'cross-linux-aarch64']
+	# ZIG
+	# FIXME(kp2pml30) check platform
+	require_relative './src/webget.rb'
+	download_to = download_dir.join('zig.tar.xz')
+	read_file(
+		uri: URI("https://ziglang.org/builds/zig-linux-x86_64-0.14.0-dev.2238+1db8cade5.tar.xz"),
+		path: download_to
+	)
+	extract_tar(download_dir.join('zig'), download_to)
+
+	# openssl
+	download_to = download_dir.join('openssl-aarch.tar.xz')
+	read_file(
+		uri: URI("http://mirror.archlinuxarm.org/aarch64/core/openssl-3.4.0-1-aarch64.pkg.tar.xz"),
+		path: download_to,
+		use_ssl: false,
+	)
+	extract_tar(download_dir.join('cross-aarch64-libs'), download_to, trim_first: false)
+end
+
 if options[:rust]
 	puts `cd "#{root}" && #{rustup} show active-toolchain || #{rustup} toolchain install`
 	run_command(rustup, 'component', 'add', 'rustfmt', chdir: root)
@@ -181,10 +209,19 @@ def load_packages_from_lists(dir)
 	run_command(rustup, 'component', 'add', '--toolchain', cur_toolchain, 'rust-std', chdir: ext_path)
 end
 
+if options[:rust] and options[:'cross-linux-aarch64']
+	ext_path = root.join('executor')
+	cur_toolchain = run_command(rustup, 'show', 'active-toolchain', chdir: ext_path)
+	cur_toolchain = cur_toolchain.lines.map { |l| l.strip }.filter { |l| l.size != 0 }.last
+	cur_toolchain = /^([a-zA-Z0-9\-_]+)/.match(cur_toolchain)[1]
+	logger.debug("installing for toolchain #{cur_toolchain}")
+	run_command(rustup, 'target', 'add', '--toolchain', cur_toolchain, 'aarch64-unknown-linux-gnu', chdir: ext_path)
+	run_command(rustup, 'component', 'add', '--toolchain', cur_toolchain, 'rust-std', chdir: ext_path)
+end
+
 if options[:wasi]
 	logger.debug("downloading runners dependencies")
-	src = Pathname.new(__FILE__).parent.join('src', 'wasi-sdk.rb').read
-	eval(src, binding)
+	require_relative './src/wasi-sdk.rb'
 
 	if find_executable('docker').nil?
 		logger.error("docker is required")
diff --git a/build-scripts/install/src/wasi-sdk.rb b/build-scripts/install/src/wasi-sdk.rb
@@ -1,77 +1,26 @@
-wasi_dir = download_dir.join('wasi-sdk-24')
+wasi_dir = $download_dir.join('wasi-sdk-24')
 
-def read_file(uri:, path:)
-	loop {
-		request = Net::HTTP::Get.new(uri)
-		Net::HTTP.start(uri.host, uri.port, :use_ssl => true) do |http|
-			http.request(request) { |response|
-				case response
-				when Net::HTTPRedirection
-					uri = URI(response['location'])
-				when Net::HTTPSuccess
-					File.open(path, 'wb') { |file|
-						response.read_body { |chunk|
-							file.write chunk
-						}
-					}
-					return
-				else
-					raise "invalid response #{response}"
-				end
-			}
-		end
-	}
-end
+require_relative './webget.rb'
 
 if wasi_dir.exist?()
-	logger.info("wasi-sdk-24 already exists")
+	$logger.info("wasi-sdk-24 already exists")
 else
-	download_to = download_dir.join('wasi-sdk-24.tar.gz')
+	download_to = $download_dir.join('wasi-sdk-24.tar.gz')
 	if not download_to.exist?
-		logger.info("downloading wasi-sdk-24 to #{download_to}")
-		Net::HTTP.start("github.com") do |http|
-			plat = case PLATFORM
-				when 'amd64'
-				'x86_64'
-			when 'aarch64'
-				'arm64'
-			else
-				raise "unsupported platform"
-			end
-			puts "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-#{plat}-#{OS}.tar.gz"
-			read_file(
-				uri: URI("https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-#{plat}-#{OS}.tar.gz"),
-				path: download_to
-			)
-		end
-	end
-	tar_extract = Gem::Package::TarReader.new(Zlib::GzipReader.open(download_to))
-	begin
-		tar_extract.rewind
-		dest = nil
-		g_name = Proc.new { |v|
-			names = v.split('/')
-			wasi_dir.join(*names[1...names.size])
-		}
-		tar_extract.each do |entry|
-			if entry.full_name == '././@LongLink'
-				dest = g_name.call(entry.read.strip)
-				next
-			end
-			dest ||= g_name.call(entry.full_name)
-			if entry.directory?
-				dest.mkpath
-			elsif entry.file?
-				File.open dest, "wb" do |f|
-					f.write entry.read
-				end
-				FileUtils.chmod entry.header.mode, dest, :verbose => false
-			elsif entry.header.typeflag == '2' #Symlink!
-				File.symlink entry.header.linkname, dest
-			end
-			dest = nil
+		$logger.info("downloading wasi-sdk-24 to #{download_to}")
+		plat = case PLATFORM
+			when 'amd64'
+			'x86_64'
+		when 'aarch64'
+			'arm64'
+		else
+			raise "unsupported platform"
 		end
-	ensure
-		tar_extract.close
+		puts "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-#{plat}-#{OS}.tar.gz"
+		read_file(
+			uri: URI("https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-#{plat}-#{OS}.tar.gz"),
+			path: download_to
+		)
 	end
+	extract_tar(wasi_dir, download_to)
 end
diff --git a/build-scripts/install/src/webget.rb b/build-scripts/install/src/webget.rb
@@ -0,0 +1,86 @@
+def read_file(uri:, path:, use_ssl: true)
+	if path.exist?
+		$logger.info("already exists #{path}, skipping download")
+		return
+	end
+	$logger.info("downloading #{uri} => #{path}")
+	loop {
+		request = Net::HTTP::Get.new(uri)
+		Net::HTTP.start(uri.host, uri.port, :use_ssl => use_ssl) do |http|
+			http.request(request) { |response|
+				case response
+				when Net::HTTPRedirection
+					uri = URI(response['location'])
+				when Net::HTTPSuccess
+					File.open(path, 'wb') { |file|
+						response.read_body { |chunk|
+							file.write chunk
+						}
+					}
+					return
+				else
+					raise "invalid response #{response}"
+				end
+			}
+		end
+	}
+end
+
+def extract_tar(base_dir, path, trim_first: true)
+	if base_dir.exist?
+		$logger.info("already exists #{base_dir}, skipping extraction")
+		return
+	end
+	base_dir.mkpath
+	if path.to_s.end_with?('.xz')
+		begin
+			Gem::Specification::find_by_name('ruby-xz')
+		rescue Gem::LoadError
+			require 'rubygems/commands/install_command'
+			cmd = Gem::Commands::InstallCommand.new
+			cmd.handle_options ['--user-install', 'ruby-xz']
+			begin
+				cmd.execute
+			rescue Gem::SystemExitException
+			end
+		end
+		require 'xz'
+		decomp = path.sub_ext('.tar')
+		XZ::decompress_file(path, decomp)
+		tar_extract = Gem::Package::TarReader.new(File.open(decomp))
+	else
+		tar_extract = Gem::Package::TarReader.new(Zlib::GzipReader.open(path))
+	end
+	begin
+		tar_extract.rewind
+		dest = nil
+		g_name = Proc.new { |v|
+			names = v.split('/')
+			if trim_first
+				base_dir.join(*names[1...names.size])
+			else
+				base_dir.join(*names)
+			end
+		}
+		tar_extract.each do |entry|
+			if entry.full_name == '././@LongLink'
+				dest = g_name.call(entry.read.strip)
+				next
+			end
+			dest ||= g_name.call(entry.full_name)
+			if entry.directory?
+				dest.mkpath
+			elsif entry.file?
+				File.open dest, "wb" do |f|
+					f.write entry.read
+				end
+				FileUtils.chmod entry.header.mode, dest, :verbose => false
+			elsif entry.header.typeflag == '2' #Symlink!
+				File.symlink entry.header.linkname, dest
+			end
+			dest = nil
+		end
+	ensure
+		tar_extract.close
+	end
+end
diff --git a/build-scripts/zig-driver.py b/build-scripts/zig-driver.py
diff --git a/executor/yabuild.rb b/executor/yabuild.rb
diff --git a/yabuild-default-conf.rb b/yabuild-default-conf.rb

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+extend_config({`
	`2`	`+ :executor_target => "aarch64-unknown-linux-gnu"`
	`3`	`+})`